Commit message: Fully incorporate the code from Python 3.7.2 (#78)

This is a full port, following the recipe in update_process.md. I've also tried to keep the recipe up to date and improved the automation (see tools/script). I haven't cleaned up the commits. As of #77 there are a few tests that sanity-check this (though it's far from a full test suite), and they're run by Travis-CI and AppVeyor.
CVE: CVE-2019-19275
CWE: CWE-125
Vulnerability description: typed_ast 1.3.0 and 1.3.1 has an ast_for_arguments out-of-bounds read. An attacker with the ability to cause a Python interpreter to parse Python source (but not necessarily execute it) may be able to crash the interpreter process. This could be a concern, for example, in a web-based service that parses (but does not execute) Python code. (This issue also affected certain Python 3.8.0-alpha prereleases.)

Project: typed_ast

Ta3Tokenizer_FindEncodingFilename(int fd, PyObject *filename)
{
    struct tok_state *tok;
    FILE *fp;
    char *p_start =NULL , *p_end =NULL , *encoding = NULL;

#ifndef PGEN
    fd = _Py_dup(fd);
#else
    fd = dup(fd);
#endif
    if (fd < 0) {
        return NULL;
    }

    fp = fdopen(fd, "r");
    if (fp == NULL) {
        return NULL;
    }
    tok = Ta3Tokenizer_FromFile(fp, NULL, NULL, NULL);
    if (tok == NULL) {
        fclose(fp);
        return NULL;
    }
#ifndef PGEN
    if (filename != NULL) {
        Py_INCREF(filename);
        tok->filename = filename;
    }
    else {
        tok->filename = PyUnicode_FromString("<string>");
        if (tok->filename == NULL) {
            fclose(fp);
            Ta3Tokenizer_Free(tok);
            return encoding;
        }
    }
#endif
    while (tok->lineno < 2 && tok->done == E_OK) {
        Ta3Tokenizer_Get(tok, &p_start, &p_end);
    }
    fclose(fp);
    if (tok->encoding) {
        encoding = (char *)PyMem_MALLOC(strlen(tok->encoding) + 1);
        if (encoding)
        strcpy(encoding, tok->encoding);
    }
    Ta3Tokenizer_Free(tok);
    return encoding;
}

Project: typed_ast

Ta3Token_TwoChars(int c1, int c2)
{
    switch (c1) {
    case '=':
        switch (c2) {
        case '=':               return EQEQUAL;
        }
        break;
    case '!':
        switch (c2) {
        case '=':               return NOTEQUAL;
        }
        break;
    case '<':
        switch (c2) {
        case '>':               return NOTEQUAL;
        case '=':               return LESSEQUAL;
        case '<':               return LEFTSHIFT;
        }
        break;
    case '>':
        switch (c2) {
        case '=':               return GREATEREQUAL;
        case '>':               return RIGHTSHIFT;
        }
        break;
    case '+':
        switch (c2) {
        case '=':               return PLUSEQUAL;
        }
        break;
    case '-':
        switch (c2) {
        case '=':               return MINEQUAL;
        case '>':               return RARROW;
        }
        break;
    case '*':
        switch (c2) {
        case '*':               return DOUBLESTAR;
        case '=':               return STAREQUAL;
        }
        break;
    case '/':
        switch (c2) {
        case '/':               return DOUBLESLASH;
        case '=':               return SLASHEQUAL;
        }
        break;
    case '|':
        switch (c2) {
        case '=':               return VBAREQUAL;
        }
        break;
    case '%':
        switch (c2) {
        case '=':               return PERCENTEQUAL;
        }
        break;
    case '&':
        switch (c2) {
        case '=':               return AMPEREQUAL;
        }
        break;
    case '^':
        switch (c2) {
        case '=':               return CIRCUMFLEXEQUAL;
        }
        break;
    case '@':
        switch (c2) {
        case '=':               return ATEQUAL;
        }
        break;
    }
    return OP;
}

Project: typed_ast

tok_new(void)
{
    struct tok_state *tok = (struct tok_state *)PyMem_MALLOC(
                                            sizeof(struct tok_state));
    if (tok == NULL)
        return NULL;
    tok->buf = tok->cur = tok->end = tok->inp = tok->start = NULL;
    tok->done = E_OK;
    tok->fp = NULL;
    tok->input = NULL;
    tok->tabsize = TABSIZE;
    tok->indent = 0;
    tok->indstack[0] = 0;

    tok->atbol = 1;
    tok->pendin = 0;
    tok->prompt = tok->nextprompt = NULL;
    tok->lineno = 0;
    tok->level = 0;
    tok->altindstack[0] = 0;
    tok->decoding_state = STATE_INIT;
    tok->decoding_erred = 0;
    tok->read_coding_spec = 0;
    tok->enc = NULL;
    tok->encoding = NULL;
    tok->cont_line = 0;
#ifndef PGEN
    tok->filename = NULL;
    tok->decoding_readline = NULL;
    tok->decoding_buffer = NULL;
#endif

    tok->async_def = 0;
    tok->async_def_indent = 0;
    tok->async_def_nl = 0;
    tok->async_always = 0;

    return tok;
}

Project: typed_ast
File: tokenizer.c

decode_str(const char *str, int exec_input, struct tok_state *tok)
{
    return new_string(str, strlen(str), tok);
}

Project: typed_ast

static int init_types(void)
{
    static int initialized;
    if (initialized) return 1;
    if (add_ast_fields() < 0) return 0;
    mod_type = make_type("mod", &AST_type, NULL, 0);
    if (!mod_type) return 0;
    if (!add_attributes(mod_type, NULL, 0)) return 0;
    Module_type = make_type("Module", mod_type, Module_fields, 2);
    if (!Module_type) return 0;
    Interactive_type = make_type("Interactive", mod_type, Interactive_fields,
                                 1);
    if (!Interactive_type) return 0;
    Expression_type = make_type("Expression", mod_type, Expression_fields, 1);
    if (!Expression_type) return 0;
    FunctionType_type = make_type("FunctionType", mod_type,
                                  FunctionType_fields, 2);
    if (!FunctionType_type) return 0;
    Suite_type = make_type("Suite", mod_type, Suite_fields, 1);
    if (!Suite_type) return 0;
    stmt_type = make_type("stmt", &AST_type, NULL, 0);
    if (!stmt_type) return 0;
    if (!add_attributes(stmt_type, stmt_attributes, 2)) return 0;
    FunctionDef_type = make_type("FunctionDef", stmt_type, FunctionDef_fields,
                                 6);
    if (!FunctionDef_type) return 0;
    AsyncFunctionDef_type = make_type("AsyncFunctionDef", stmt_type,
                                      AsyncFunctionDef_fields, 6);
    if (!AsyncFunctionDef_type) return 0;
    ClassDef_type = make_type("ClassDef", stmt_type, ClassDef_fields, 5);
    if (!ClassDef_type) return 0;
    Return_type = make_type("Return", stmt_type, Return_fields, 1);
    if (!Return_type) return 0;
    Delete_type = make_type("Delete", stmt_type, Delete_fields, 1);
    if (!Delete_type) return 0;
    Assign_type = make_type("Assign", stmt_type, Assign_fields, 3);
    if (!Assign_type) return 0;
    AugAssign_type = make_type("AugAssign", stmt_type, AugAssign_fields, 3);
    if (!AugAssign_type) return 0;
    AnnAssign_type = make_type("AnnAssign", stmt_type, AnnAssign_fields, 4);
    if (!AnnAssign_type) return 0;
    For_type = make_type("For", stmt_type, For_fields, 5);
    if (!For_type) return 0;
    AsyncFor_type = make_type("AsyncFor", stmt_type, AsyncFor_fields, 5);
    if (!AsyncFor_type) return 0;
    While_type = make_type("While", stmt_type, While_fields, 3);
    if (!While_type) return 0;
    If_type = make_type("If", stmt_type, If_fields, 3);
    if (!If_type) return 0;
    With_type = make_type("With", stmt_type, With_fields, 3);
    if (!With_type) return 0;
    AsyncWith_type = make_type("AsyncWith", stmt_type, AsyncWith_fields, 3);
    if (!AsyncWith_type) return 0;
    Raise_type = make_type("Raise", stmt_type, Raise_fields, 2);
    if (!Raise_type) return 0;
    Try_type = make_type("Try", stmt_type, Try_fields, 4);
    if (!Try_type) return 0;
    Assert_type = make_type("Assert", stmt_type, Assert_fields, 2);
    if (!Assert_type) return 0;
    Import_type = make_type("Import", stmt_type, Import_fields, 1);
    if (!Import_type) return 0;
    ImportFrom_type = make_type("ImportFrom", stmt_type, ImportFrom_fields, 3);
    if (!ImportFrom_type) return 0;
    Global_type = make_type("Global", stmt_type, Global_fields, 1);
    if (!Global_type) return 0;
    Nonlocal_type = make_type("Nonlocal", stmt_type, Nonlocal_fields, 1);
    if (!Nonlocal_type) return 0;
    Expr_type = make_type("Expr", stmt_type, Expr_fields, 1);
    if (!Expr_type) return 0;
    Pass_type = make_type("Pass", stmt_type, NULL, 0);
    if (!Pass_type) return 0;
    Break_type = make_type("Break", stmt_type, NULL, 0);
    if (!Break_type) return 0;
    Continue_type = make_type("Continue", stmt_type, NULL, 0);
    if (!Continue_type) return 0;
    expr_type = make_type("expr", &AST_type, NULL, 0);
    if (!expr_type) return 0;
    if (!add_attributes(expr_type, expr_attributes, 2)) return 0;
    BoolOp_type = make_type("BoolOp", expr_type, BoolOp_fields, 2);
    if (!BoolOp_type) return 0;
    BinOp_type = make_type("BinOp", expr_type, BinOp_fields, 3);
    if (!BinOp_type) return 0;
    UnaryOp_type = make_type("UnaryOp", expr_type, UnaryOp_fields, 2);
    if (!UnaryOp_type) return 0;
    Lambda_type = make_type("Lambda", expr_type, Lambda_fields, 2);
    if (!Lambda_type) return 0;
    IfExp_type = make_type("IfExp", expr_type, IfExp_fields, 3);
    if (!IfExp_type) return 0;
    Dict_type = make_type("Dict", expr_type, Dict_fields, 2);
    if (!Dict_type) return 0;
    Set_type = make_type("Set", expr_type, Set_fields, 1);
    if (!Set_type) return 0;
    ListComp_type = make_type("ListComp", expr_type, ListComp_fields, 2);
    if (!ListComp_type) return 0;
    SetComp_type = make_type("SetComp", expr_type, SetComp_fields, 2);
    if (!SetComp_type) return 0;
    DictComp_type = make_type("DictComp", expr_type, DictComp_fields, 3);
    if (!DictComp_type) return 0;
    GeneratorExp_type = make_type("GeneratorExp", expr_type,
                                  GeneratorExp_fields, 2);
    if (!GeneratorExp_type) return 0;
    Await_type = make_type("Await", expr_type, Await_fields, 1);
    if (!Await_type) return 0;
    Yield_type = make_type("Yield", expr_type, Yield_fields, 1);
    if (!Yield_type) return 0;
    YieldFrom_type = make_type("YieldFrom", expr_type, YieldFrom_fields, 1);
    if (!YieldFrom_type) return 0;
    Compare_type = make_type("Compare", expr_type, Compare_fields, 3);
    if (!Compare_type) return 0;
    Call_type = make_type("Call", expr_type, Call_fields, 3);
    if (!Call_type) return 0;
    Num_type = make_type("Num", expr_type, Num_fields, 1);
    if (!Num_type) return 0;
    Str_type = make_type("Str", expr_type, Str_fields, 2);
    if (!Str_type) return 0;
    FormattedValue_type = make_type("FormattedValue", expr_type,
                                    FormattedValue_fields, 3);
    if (!FormattedValue_type) return 0;
    JoinedStr_type = make_type("JoinedStr", expr_type, JoinedStr_fields, 1);
    if (!JoinedStr_type) return 0;
    Bytes_type = make_type("Bytes", expr_type, Bytes_fields, 1);
    if (!Bytes_type) return 0;
    NameConstant_type = make_type("NameConstant", expr_type,
                                  NameConstant_fields, 1);
    if (!NameConstant_type) return 0;
    Ellipsis_type = make_type("Ellipsis", expr_type, NULL, 0);
    if (!Ellipsis_type) return 0;
    Constant_type = make_type("Constant", expr_type, Constant_fields, 1);
    if (!Constant_type) return 0;
    Attribute_type = make_type("Attribute", expr_type, Attribute_fields, 3);
    if (!Attribute_type) return 0;
    Subscript_type = make_type("Subscript", expr_type, Subscript_fields, 3);
    if (!Subscript_type) return 0;
    Starred_type = make_type("Starred", expr_type, Starred_fields, 2);
    if (!Starred_type) return 0;
    Name_type = make_type("Name", expr_type, Name_fields, 2);
    if (!Name_type) return 0;
    List_type = make_type("List", expr_type, List_fields, 2);
    if (!List_type) return 0;
    Tuple_type = make_type("Tuple", expr_type, Tuple_fields, 2);
    if (!Tuple_type) return 0;
    expr_context_type = make_type("expr_context", &AST_type, NULL, 0);
    if (!expr_context_type) return 0;
    if (!add_attributes(expr_context_type, NULL, 0)) return 0;
    Load_type = make_type("Load", expr_context_type, NULL, 0);
    if (!Load_type) return 0;
    Load_singleton = PyType_GenericNew(Load_type, NULL, NULL);
    if (!Load_singleton) return 0;
    Store_type = make_type("Store", expr_context_type, NULL, 0);
    if (!Store_type) return 0;
    Store_singleton = PyType_GenericNew(Store_type, NULL, NULL);
    if (!Store_singleton) return 0;
    Del_type = make_type("Del", expr_context_type, NULL, 0);
    if (!Del_type) return 0;
    Del_singleton = PyType_GenericNew(Del_type, NULL, NULL);
    if (!Del_singleton) return 0;
    AugLoad_type = make_type("AugLoad", expr_context_type, NULL, 0);
    if (!AugLoad_type) return 0;
    AugLoad_singleton = PyType_GenericNew(AugLoad_type, NULL, NULL);
    if (!AugLoad_singleton) return 0;
    AugStore_type = make_type("AugStore", expr_context_type, NULL, 0);
    if (!AugStore_type) return 0;
    AugStore_singleton = PyType_GenericNew(AugStore_type, NULL, NULL);
    if (!AugStore_singleton) return 0;
    Param_type = make_type("Param", expr_context_type, NULL, 0);
    if (!Param_type) return 0;
    Param_singleton = PyType_GenericNew(Param_type, NULL, NULL);
    if (!Param_singleton) return 0;
    slice_type = make_type("slice", &AST_type, NULL, 0);
    if (!slice_type) return 0;
    if (!add_attributes(slice_type, NULL, 0)) return 0;
    Slice_type = make_type("Slice", slice_type, Slice_fields, 3);
    if (!Slice_type) return 0;
    ExtSlice_type = make_type("ExtSlice", slice_type, ExtSlice_fields, 1);
    if (!ExtSlice_type) return 0;
    Index_type = make_type("Index", slice_type, Index_fields, 1);
    if (!Index_type) return 0;
    boolop_type = make_type("boolop", &AST_type, NULL, 0);
    if (!boolop_type) return 0;
    if (!add_attributes(boolop_type, NULL, 0)) return 0;
    And_type = make_type("And", boolop_type, NULL, 0);
    if (!And_type) return 0;
    And_singleton = PyType_GenericNew(And_type, NULL, NULL);
    if (!And_singleton) return 0;
    Or_type = make_type("Or", boolop_type, NULL, 0);
    if (!Or_type) return 0;
    Or_singleton = PyType_GenericNew(Or_type, NULL, NULL);
    if (!Or_singleton) return 0;
    operator_type = make_type("operator", &AST_type, NULL, 0);
    if (!operator_type) return 0;
    if (!add_attributes(operator_type, NULL, 0)) return 0;
    Add_type = make_type("Add", operator_type, NULL, 0);
    if (!Add_type) return 0;
    Add_singleton = PyType_GenericNew(Add_type, NULL, NULL);
    if (!Add_singleton) return 0;
    Sub_type = make_type("Sub", operator_type, NULL, 0);
    if (!Sub_type) return 0;
    Sub_singleton = PyType_GenericNew(Sub_type, NULL, NULL);
    if (!Sub_singleton) return 0;
    Mult_type = make_type("Mult", operator_type, NULL, 0);
    if (!Mult_type) return 0;
    Mult_singleton = PyType_GenericNew(Mult_type, NULL, NULL);
    if (!Mult_singleton) return 0;
    MatMult_type = make_type("MatMult", operator_type, NULL, 0);
    if (!MatMult_type) return 0;
    MatMult_singleton = PyType_GenericNew(MatMult_type, NULL, NULL);
    if (!MatMult_singleton) return 0;
    Div_type = make_type("Div", operator_type, NULL, 0);
    if (!Div_type) return 0;
    Div_singleton = PyType_GenericNew(Div_type, NULL, NULL);
    if (!Div_singleton) return 0;
    Mod_type = make_type("Mod", operator_type, NULL, 0);
    if (!Mod_type) return 0;
    Mod_singleton = PyType_GenericNew(Mod_type, NULL, NULL);
    if (!Mod_singleton) return 0;
    Pow_type = make_type("Pow", operator_type, NULL, 0);
    if (!Pow_type) return 0;
    Pow_singleton = PyType_GenericNew(Pow_type, NULL, NULL);
    if (!Pow_singleton) return 0;
    LShift_type = make_type("LShift", operator_type, NULL, 0);
    if (!LShift_type) return 0;
    LShift_singleton = PyType_GenericNew(LShift_type, NULL, NULL);
    if (!LShift_singleton) return 0;
    RShift_type = make_type("RShift", operator_type, NULL, 0);
    if (!RShift_type) return 0;
    RShift_singleton = PyType_GenericNew(RShift_type, NULL, NULL);
    if (!RShift_singleton) return 0;
    BitOr_type = make_type("BitOr", operator_type, NULL, 0);
    if (!BitOr_type) return 0;
    BitOr_singleton = PyType_GenericNew(BitOr_type, NULL, NULL);
    if (!BitOr_singleton) return 0;
    BitXor_type = make_type("BitXor", operator_type, NULL, 0);
    if (!BitXor_type) return 0;
    BitXor_singleton = PyType_GenericNew(BitXor_type, NULL, NULL);
    if (!BitXor_singleton) return 0;
    BitAnd_type = make_type("BitAnd", operator_type, NULL, 0);
    if (!BitAnd_type) return 0;
    BitAnd_singleton = PyType_GenericNew(BitAnd_type, NULL, NULL);
    if (!BitAnd_singleton) return 0;
    FloorDiv_type = make_type("FloorDiv", operator_type, NULL, 0);
    if (!FloorDiv_type) return 0;
    FloorDiv_singleton = PyType_GenericNew(FloorDiv_type, NULL, NULL);
    if (!FloorDiv_singleton) return 0;
    unaryop_type = make_type("unaryop", &AST_type, NULL, 0);
    if (!unaryop_type) return 0;
    if (!add_attributes(unaryop_type, NULL, 0)) return 0;
    Invert_type = make_type("Invert", unaryop_type, NULL, 0);
    if (!Invert_type) return 0;
    Invert_singleton = PyType_GenericNew(Invert_type, NULL, NULL);
    if (!Invert_singleton) return 0;
    Not_type = make_type("Not", unaryop_type, NULL, 0);
    if (!Not_type) return 0;
    Not_singleton = PyType_GenericNew(Not_type, NULL, NULL);
    if (!Not_singleton) return 0;
    UAdd_type = make_type("UAdd", unaryop_type, NULL, 0);
    if (!UAdd_type) return 0;
    UAdd_singleton = PyType_GenericNew(UAdd_type, NULL, NULL);
    if (!UAdd_singleton) return 0;
    USub_type = make_type("USub", unaryop_type, NULL, 0);
    if (!USub_type) return 0;
    USub_singleton = PyType_GenericNew(USub_type, NULL, NULL);
    if (!USub_singleton) return 0;
    cmpop_type = make_type("cmpop", &AST_type, NULL, 0);
    if (!cmpop_type) return 0;
    if (!add_attributes(cmpop_type, NULL, 0)) return 0;
    Eq_type = make_type("Eq", cmpop_type, NULL, 0);
    if (!Eq_type) return 0;
    Eq_singleton = PyType_GenericNew(Eq_type, NULL, NULL);
    if (!Eq_singleton) return 0;
    NotEq_type = make_type("NotEq", cmpop_type, NULL, 0);
    if (!NotEq_type) return 0;
    NotEq_singleton = PyType_GenericNew(NotEq_type, NULL, NULL);
    if (!NotEq_singleton) return 0;
    Lt_type = make_type("Lt", cmpop_type, NULL, 0);
    if (!Lt_type) return 0;
    Lt_singleton = PyType_GenericNew(Lt_type, NULL, NULL);
    if (!Lt_singleton) return 0;
    LtE_type = make_type("LtE", cmpop_type, NULL, 0);
    if (!LtE_type) return 0;
    LtE_singleton = PyType_GenericNew(LtE_type, NULL, NULL);
    if (!LtE_singleton) return 0;
    Gt_type = make_type("Gt", cmpop_type, NULL, 0);
    if (!Gt_type) return 0;
    Gt_singleton = PyType_GenericNew(Gt_type, NULL, NULL);
    if (!Gt_singleton) return 0;
    GtE_type = make_type("GtE", cmpop_type, NULL, 0);
    if (!GtE_type) return 0;
    GtE_singleton = PyType_GenericNew(GtE_type, NULL, NULL);
    if (!GtE_singleton) return 0;
    Is_type = make_type("Is", cmpop_type, NULL, 0);
    if (!Is_type) return 0;
    Is_singleton = PyType_GenericNew(Is_type, NULL, NULL);
    if (!Is_singleton) return 0;
    IsNot_type = make_type("IsNot", cmpop_type, NULL, 0);
    if (!IsNot_type) return 0;
    IsNot_singleton = PyType_GenericNew(IsNot_type, NULL, NULL);
    if (!IsNot_singleton) return 0;
    In_type = make_type("In", cmpop_type, NULL, 0);
    if (!In_type) return 0;
    In_singleton = PyType_GenericNew(In_type, NULL, NULL);
    if (!In_singleton) return 0;
    NotIn_type = make_type("NotIn", cmpop_type, NULL, 0);
    if (!NotIn_type) return 0;
    NotIn_singleton = PyType_GenericNew(NotIn_type, NULL, NULL);
    if (!NotIn_singleton) return 0;
    comprehension_type = make_type("comprehension", &AST_type,
                                   comprehension_fields, 4);
    if (!comprehension_type) return 0;
    if (!add_attributes(comprehension_type, NULL, 0)) return 0;
    excepthandler_type = make_type("excepthandler", &AST_type, NULL, 0);
    if (!excepthandler_type) return 0;
    if (!add_attributes(excepthandler_type, excepthandler_attributes, 2))
        return 0;
    ExceptHandler_type = make_type("ExceptHandler", excepthandler_type,
                                   ExceptHandler_fields, 3);
    if (!ExceptHandler_type) return 0;
    arguments_type = make_type("arguments", &AST_type, arguments_fields, 6);
    if (!arguments_type) return 0;
    if (!add_attributes(arguments_type, NULL, 0)) return 0;
    arg_type = make_type("arg", &AST_type, arg_fields, 3);
    if (!arg_type) return 0;
    if (!add_attributes(arg_type, arg_attributes, 2)) return 0;
    keyword_type = make_type("keyword", &AST_type, keyword_fields, 2);
    if (!keyword_type) return 0;
    if (!add_attributes(keyword_type, NULL, 0)) return 0;
    alias_type = make_type("alias", &AST_type, alias_fields, 2);
    if (!alias_type) return 0;
    if (!add_attributes(alias_type, NULL, 0)) return 0;
    withitem_type = make_type("withitem", &AST_type, withitem_fields, 2);
    if (!withitem_type) return 0;
    if (!add_attributes(withitem_type, NULL, 0)) return 0;
    type_ignore_type = make_type("type_ignore", &AST_type, NULL, 0);
    if (!type_ignore_type) return 0;
    if (!add_attributes(type_ignore_type, NULL, 0)) return 0;
    TypeIgnore_type = make_type("TypeIgnore", type_ignore_type,
                                TypeIgnore_fields, 1);
    if (!TypeIgnore_type) return 0;
    initialized = 1;
    return 1;
}

Project: typed_ast

Ellipsis(int lineno, int col_offset, PyArena *arena)
{
    expr_ty p;
    p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->kind = Ellipsis_kind;
    p->lineno = lineno;
    p->col_offset = col_offset;
    return p;
}

Project: typed_ast
File: tokenizer.c

indenterror(struct tok_state *tok)
{
    tok->done = E_TABSPACE;
    tok->cur = tok->inp;
    return ERRORTOKEN;
}

Project: typed_ast

Name(identifier id, expr_context_ty ctx, int lineno, int col_offset, PyArena
     *arena)
{
    expr_ty p;
    if (!id) {
        PyErr_SetString(PyExc_ValueError,
                        "field id is required for Name");
        return NULL;
    }
    if (!ctx) {
        PyErr_SetString(PyExc_ValueError,
                        "field ctx is required for Name");
        return NULL;
    }
    p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->kind = Name_kind;
    p->v.Name.id = id;
    p->v.Name.ctx = ctx;
    p->lineno = lineno;
    p->col_offset = col_offset;
    return p;
}

Project: typed_ast

AnnAssign(expr_ty target, expr_ty annotation, expr_ty value, int simple, int
          lineno, int col_offset, PyArena *arena)
{
    stmt_ty p;
    if (!target) {
        PyErr_SetString(PyExc_ValueError,
                        "field target is required for AnnAssign");
        return NULL;
    }
    if (!annotation) {
        PyErr_SetString(PyExc_ValueError,
                        "field annotation is required for AnnAssign");
        return NULL;
    }
    p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->kind = AnnAssign_kind;
    p->v.AnnAssign.target = target;
    p->v.AnnAssign.annotation = annotation;
    p->v.AnnAssign.value = value;
    p->v.AnnAssign.simple = simple;
    p->lineno = lineno;
    p->col_offset = col_offset;
    return p;
}

Project: typed_ast
File: Python-ast.c

static int obj2ast_object(PyObject* obj, PyObject** out, PyArena* arena)
{
    if (obj == Py_None)
        obj = NULL;
    if (obj) {
        if (PyArena_AddPyObject(arena, obj) < 0) {
            *out = NULL;
            return -1;
        }
        Py_INCREF(obj);
    }
    *out = obj;
    return 0;
}

Project: typed_ast

ast_for_expr_stmt(struct compiling *c, const node *n)
{
    int num;
    REQ(n, expr_stmt);
    /* expr_stmt: testlist (augassign (yield_expr|testlist)
                | ('=' (yield_expr|testlist))* [TYPE_COMMENT])
       testlist: test (',' test)* [',']
       augassign: '+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^='
                | '<<=' | '>>=' | '**=' | '//='
       test: ... here starts the operator precendence dance
     */
    num = NCH(n);

    if (num == 1 || (num == 2 && TYPE(CHILD(n, 1)) == TYPE_COMMENT)) {
        expr_ty e = ast_for_testlist(c, CHILD(n, 0));
        if (!e)
            return NULL;

        return Expr(e, LINENO(n), n->n_col_offset, c->c_arena);
    }
    else if (TYPE(CHILD(n, 1)) == augassign) {
        expr_ty expr1, expr2;
        operator_ty newoperator;
        node *ch = CHILD(n, 0);

        expr1 = ast_for_testlist(c, ch);
        if (!expr1)
            return NULL;
        if(!set_context(c, expr1, Store, ch))
            return NULL;
        /* set_context checks that most expressions are not the left side.
          Augmented assignments can only have a name, a subscript, or an
          attribute on the left, though, so we have to explicitly check for
          those. */
        switch (expr1->kind) {
            case Name_kind:
            case Attribute_kind:
            case Subscript_kind:
                break;
            default:
                ast_error(ch, "illegal expression for augmented assignment");
                return NULL;
        }

        ch = CHILD(n, 2);
        if (TYPE(ch) == testlist)
            expr2 = ast_for_testlist(c, ch);
        else
            expr2 = ast_for_expr(c, ch);
        if (!expr2)
            return NULL;

        newoperator = ast_for_augassign(c, CHILD(n, 1));
        if (!newoperator)
            return NULL;

        return AugAssign(expr1, newoperator, expr2, LINENO(n), n->n_col_offset,
                         c->c_arena);
    }
    else {
        int i, nch_minus_type, has_type_comment;
        asdl_seq *targets;
        node *value;
        expr_ty expression;
        string type_comment;

        /* a normal assignment */
        REQ(CHILD(n, 1), EQUAL);

        has_type_comment = TYPE(CHILD(n, num - 1)) == TYPE_COMMENT;
        nch_minus_type = num - has_type_comment;

        targets = asdl_seq_new(nch_minus_type / 2, c->c_arena);
        if (!targets)
            return NULL;
        for (i = 0; i < nch_minus_type - 2; i += 2) {
            expr_ty e;
            node *ch = CHILD(n, i);
            if (TYPE(ch) == yield_expr) {
                ast_error(ch, "assignment to yield expression not possible");
                return NULL;
            }
            e = ast_for_testlist(c, ch);
            if (!e)
                return NULL;

            /* set context to assign */
            if (!set_context(c, e, Store, CHILD(n, i)))
                return NULL;

            asdl_seq_SET(targets, i / 2, e);
        }
        value = CHILD(n, nch_minus_type - 1);
        if (TYPE(value) == testlist)
            expression = ast_for_testlist(c, value);
        else
            expression = ast_for_expr(c, value);
        if (!expression)
            return NULL;
        if (has_type_comment)
            type_comment = NEW_TYPE_COMMENT(CHILD(n, nch_minus_type));
        else
            type_comment = NULL;
        return Assign(targets, expression, type_comment, LINENO(n), n->n_col_offset,
                      c->c_arena);
    }
}

Project: typed_ast
File: parsetok.c

initerr(perrdetail *err_ret, PyObject *filename)
{
    err_ret->error = E_OK;
    err_ret->lineno = 0;
    err_ret->offset = 0;
    err_ret->text = NULL;
    err_ret->token = -1;
    err_ret->expected = -1;
#ifndef PGEN
    if (filename) {
        Py_INCREF(filename);
        err_ret->filename = filename;
    }
    else {
        err_ret->filename = PyUnicode_FromString("<string>");
        if (err_ret->filename == NULL) {
            err_ret->error = E_ERROR;
            return -1;
        }
    }
#endif
    return 0;
}

Project: typed_ast

parsestrplus(struct compiling *c, const node *n)
{
        PyObject *v;
        int i;
        REQ(CHILD(n, 0), STRING);
        if ((v = parsestr(c, n, STR(CHILD(n, 0)))) != NULL) {
                /* String literal concatenation */
                for (i = 1; i < NCH(n); i++) {
                        PyObject *s;
                        s = parsestr(c, n, STR(CHILD(n, i)));
                        if (s == NULL)
                                goto onError;
                        if (PyBytes_Check(v) && PyBytes_Check(s)) {
                                PyBytes_ConcatAndDel(&v, s);
                                if (v == NULL)
                                    goto onError;
                        }
#ifdef Py_USING_UNICODE
                        else {
                                PyObject *temp;
                                /* Python 2's PyUnicode_FromObject (which is
                                 * called on the arguments to PyUnicode_Concat)
                                 * automatically converts Bytes objects into
                                 * Str objects, but in Python 3 it throws a
                                 * syntax error.  To allow mixed literal
                                 * concatenation e.g. "foo" u"bar" (which is
                                 * valid in Python 2), we have to explicitly
                                 * check for Bytes and convert manually. */
                                if (PyBytes_Check(s)) {
                                    temp = PyUnicode_FromEncodedObject(s, NULL, "strict");
                                    Py_DECREF(s);
                                    s = temp;
                                }

                                if (PyBytes_Check(v)) {
                                    temp = PyUnicode_FromEncodedObject(v, NULL, "strict");
                                    Py_DECREF(v);
                                    v = temp;
                                }

                                temp = PyUnicode_Concat(v, s);
                                Py_DECREF(s);
                                Py_DECREF(v);
                                v = temp;
                                if (v == NULL)
                                    goto onError;
                        }
#endif
                }
        }
        return v;

 onError:
        Py_XDECREF(v);
        return NULL;
}

Project: typed_ast

tok_dump(int type, char *start, char *end)
{
    printf("%s", _Ta3Parser_TokenNames[type]);
    if (type == NAME || type == NUMBER || type == STRING || type == OP)
        printf("(%.*s)", (int)(end - start), start);
}

Project: typed_ast
File: tokenizer.c

check_bom(int get_char(struct tok_state *),
          void unget_char(int, struct tok_state *),
          int set_readline(struct tok_state *, const char *),
          struct tok_state *tok)
{
    int ch1, ch2, ch3;
    ch1 = get_char(tok);
    tok->decoding_state = STATE_RAW;
    if (ch1 == EOF) {
        return 1;
    } else if (ch1 == 0xEF) {
        ch2 = get_char(tok);
        if (ch2 != 0xBB) {
            unget_char(ch2, tok);
            unget_char(ch1, tok);
            return 1;
        }
        ch3 = get_char(tok);
        if (ch3 != 0xBF) {
            unget_char(ch3, tok);
            unget_char(ch2, tok);
            unget_char(ch1, tok);
            return 1;
        }
#if 0
    /* Disable support for UTF-16 BOMs until a decision
       is made whether this needs to be supported.  */
    } else if (ch1 == 0xFE) {
        ch2 = get_char(tok);
        if (ch2 != 0xFF) {
            unget_char(ch2, tok);
            unget_char(ch1, tok);
            return 1;
        }
        if (!set_readline(tok, "utf-16-be"))
            return 0;
        tok->decoding_state = STATE_NORMAL;
    } else if (ch1 == 0xFF) {
        ch2 = get_char(tok);
        if (ch2 != 0xFE) {
            unget_char(ch2, tok);
            unget_char(ch1, tok);
            return 1;
        }
        if (!set_readline(tok, "utf-16-le"))
            return 0;
        tok->decoding_state = STATE_NORMAL;
#endif
    } else {
        unget_char(ch1, tok);
        return 1;
    }
    if (tok->encoding != NULL)
        PyMem_FREE(tok->encoding);
    tok->encoding = new_string("utf-8", 5, tok);
    if (!tok->encoding)
        return 0;
    /* No need to set_readline: input is already utf-8 */
    return 1;
}

Project: typed_ast

set_context(struct compiling *c, expr_ty e, expr_context_ty ctx, const node *n)
{
    asdl_seq *s = NULL;
    /* If a particular expression type can't be used for assign / delete,
       set expr_name to its name and an error message will be generated.
    */
    const char* expr_name = NULL;

    /* The ast defines augmented store and load contexts, but the
       implementation here doesn't actually use them.  The code may be
       a little more complex than necessary as a result.  It also means
       that expressions in an augmented assignment have a Store context.
       Consider restructuring so that augmented assignment uses
       set_context(), too.
    */
    assert(ctx != AugStore && ctx != AugLoad);

    switch (e->kind) {
        case Attribute_kind:
            if (ctx == Store && !forbidden_check(c, n,
                                PyUnicode_AsUTF8(e->v.Attribute.attr)))
                    return 0;
            e->v.Attribute.ctx = ctx;
            break;
        case Subscript_kind:
            e->v.Subscript.ctx = ctx;
            break;
        case Name_kind:
            if (ctx == Store && !forbidden_check(c, n,
                                PyUnicode_AsUTF8(e->v.Name.id)))
                    return 0;
            e->v.Name.ctx = ctx;
            break;
        case List_kind:
            e->v.List.ctx = ctx;
            s = e->v.List.elts;
            break;
        case Tuple_kind:
            if (asdl_seq_LEN(e->v.Tuple.elts))  {
                e->v.Tuple.ctx = ctx;
                s = e->v.Tuple.elts;
            }
            else {
                expr_name = "()";
            }
            break;
        case Lambda_kind:
            expr_name = "lambda";
            break;
        case Call_kind:
            expr_name = "function call";
            break;
        case BoolOp_kind:
        case BinOp_kind:
        case UnaryOp_kind:
            expr_name = "operator";
            break;
        case GeneratorExp_kind:
            expr_name = "generator expression";
            break;
        case Yield_kind:
            expr_name = "yield expression";
            break;
        case ListComp_kind:
            expr_name = "list comprehension";
            break;
        case SetComp_kind:
            expr_name = "set comprehension";
            break;
        case DictComp_kind:
            expr_name = "dict comprehension";
            break;
        case Dict_kind:
        case Set_kind:
        case Num_kind:
        case Str_kind:
            expr_name = "literal";
            break;
        case Compare_kind:
            expr_name = "comparison";
            break;
        case Repr_kind:
            expr_name = "repr";
            break;
        case IfExp_kind:
            expr_name = "conditional expression";
            break;
        default:
            PyErr_Format(PyExc_SystemError,
                         "unexpected expression in assignment %d (line %d)",
                         e->kind, e->lineno);
            return 0;
    }
    /* Check for error string set by switch */
    if (expr_name) {
        char buf[300];
        PyOS_snprintf(buf, sizeof(buf),
                      "can't %s %s",
                      ctx == Store ? "assign to" : "delete",
                      expr_name);
        return ast_error(n, buf);
    }

    /* If the LHS is a list or tuple, we need to set the assignment
       context for all the contained elements.
    */
    if (s) {
        int i;

        for (i = 0; i < asdl_seq_LEN(s); i++) {
            if (!set_context(c, (expr_ty)asdl_seq_GET(s, i), ctx, n))
                return 0;
        }
    }
    return 1;
}

Project: typed_ast
File: Python-ast.c

PyObject* ast2obj_boolop(boolop_ty o)
{
    switch(o) {
        case And:
            Py_INCREF(And_singleton);
            return And_singleton;
        case Or:
            Py_INCREF(Or_singleton);
            return Or_singleton;
        default:
            /* should never happen, but just in case ... */
            PyErr_Format(PyExc_SystemError, "unknown boolop found");
            return NULL;
    }
}

Project: typed_ast

Ta3Tokenizer_FromUTF8(const char *str, int exec_input)
{
    struct tok_state *tok = tok_new();
    if (tok == NULL)
        return NULL;
#ifndef PGEN
    tok->input = str = translate_newlines(str, exec_input, tok);
#endif
    if (str == NULL) {
        Ta3Tokenizer_Free(tok);
        return NULL;
    }
    tok->decoding_state = STATE_RAW;
    tok->read_coding_spec = 1;
    tok->enc = NULL;
    tok->str = str;
    tok->encoding = (char *)PyMem_MALLOC(6);
    if (!tok->encoding) {
        Ta3Tokenizer_Free(tok);
        return NULL;
    }
    strcpy(tok->encoding, "utf-8");

    /* XXX: constify members. */
    tok->buf = tok->cur = tok->end = tok->inp = (char*)str;
    return tok;
}

Project: typed_ast

static PyTypeObject* make_type(char *type, PyTypeObject* base, char**fields, int num_fields)
{
    _Py_IDENTIFIER(__module__);
    _Py_IDENTIFIER(_ast3);
    PyObject *fnames, *result;
    int i;
    fnames = PyTuple_New(num_fields);
    if (!fnames) return NULL;
    for (i = 0; i < num_fields; i++) {
        PyObject *field = PyUnicode_FromString(fields[i]);
        if (!field) {
            Py_DECREF(fnames);
            return NULL;
        }
        PyTuple_SET_ITEM(fnames, i, field);
    }
    result = PyObject_CallFunction((PyObject*)&PyType_Type, "s(O){OOOO}",
                    type, base,
                    _PyUnicode_FromId(&PyId__fields), fnames,
                    _PyUnicode_FromId(&PyId___module__),
                    _PyUnicode_FromId(&PyId__ast3));
    Py_DECREF(fnames);
    return (PyTypeObject*)result;
}

Project: typed_ast

ast_for_decorator(struct compiling *c, const node *n)
{
    /* decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE */
    expr_ty d = NULL;
    expr_ty name_expr;

    REQ(n, decorator);
    REQ(CHILD(n, 0), AT);
    REQ(RCHILD(n, -1), NEWLINE);

    name_expr = ast_for_dotted_name(c, CHILD(n, 1));
    if (!name_expr)
        return NULL;

    if (NCH(n) == 3) { /* No arguments */
        d = name_expr;
        name_expr = NULL;
    }
    else if (NCH(n) == 5) { /* Call with no arguments */
        d = Call(name_expr, NULL, NULL, NULL, NULL, LINENO(n),
                 n->n_col_offset, c->c_arena);
        if (!d)
            return NULL;
        name_expr = NULL;
    }
    else {
        d = ast_for_call(c, CHILD(n, 3), name_expr);
        if (!d)
            return NULL;
        name_expr = NULL;
    }

    return d;
}

Project: typed_ast

string_object_to_c_ast(const char *s, PyObject *filename, int start,
                             PyCompilerFlags *flags, int feature_version,
                             PyArena *arena)
{
    mod_ty mod;
    PyCompilerFlags localflags;
    perrdetail err;
    int iflags = PARSER_FLAGS(flags);
    node *n;

    if (feature_version >= 7)
        iflags |= PyPARSE_ASYNC_ALWAYS;
    n = Ta3Parser_ParseStringObject(s, filename,
                                    &_Ta3Parser_Grammar, start, &err,
                                    &iflags);
    if (flags == NULL) {
        localflags.cf_flags = 0;
        flags = &localflags;
    }
    if (n) {
        flags->cf_flags |= iflags & PyCF_MASK;
        mod = Ta3AST_FromNodeObject(n, flags, filename, feature_version, arena);
        Ta3Node_Free(n);
    }
    else {
        err_input(&err);
        mod = NULL;
    }
    err_free(&err);
    return mod;
}

Project: typed_ast

obj2ast_keyword(PyObject* obj, keyword_ty* out, PyArena* arena)
{
    PyObject* tmp = NULL;
    identifier arg;
    expr_ty value;

    if (lookup_attr_id(obj, &PyId_arg, &tmp) < 0) {
        return 1;
    }
    if (tmp == NULL || tmp == Py_None) {
        Py_CLEAR(tmp);
        arg = NULL;
    }
    else {
        int res;
        res = obj2ast_identifier(tmp, &arg, arena);
        if (res != 0) goto failed;
        Py_CLEAR(tmp);
    }
    if (lookup_attr_id(obj, &PyId_value, &tmp) < 0) {
        return 1;
    }
    if (tmp == NULL) {
        PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from keyword");
        return 1;
    }
    else {
        int res;
        res = obj2ast_expr(tmp, &value, arena);
        if (res != 0) goto failed;
        Py_CLEAR(tmp);
    }
    *out = keyword(arg, value, arena);
    return 0;
failed:
    Py_XDECREF(tmp);
    return 1;
}

Project: typed_ast
File: Python-ast.c

Index(expr_ty value, PyArena *arena)
{
    slice_ty p;
    if (!value) {
        PyErr_SetString(PyExc_ValueError,
                        "field value is required for Index");
        return NULL;
    }
    p = (slice_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->kind = Index_kind;
    p->v.Index.value = value;
    return p;
}

Project: typed_ast

Continue(int lineno, int col_offset, PyArena *arena)
{
    stmt_ty p;
    p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->kind = Continue_kind;
    p->lineno = lineno;
    p->col_offset = col_offset;
    return p;
}

Project: typed_ast

ast_for_funcdef_impl(struct compiling *c, const node *n0,
                     asdl_seq *decorator_seq, bool is_async)
{
    /* funcdef: 'def' NAME parameters ['->' test] ':' [TYPE_COMMENT] suite */
    const node * const n = is_async ? CHILD(n0, 1) : n0;
    identifier name;
    arguments_ty args;
    asdl_seq *body;
    expr_ty returns = NULL;
    int name_i = 1;
    node *tc;
    string type_comment = NULL;

    if (is_async && c->c_feature_version < 5) {
        ast_error(c, n,
                  "Async functions are only supported in Python 3.5 and greater");
        return NULL;
    }

    REQ(n, funcdef);

    name = NEW_IDENTIFIER(CHILD(n, name_i));
    if (!name)
        return NULL;
    if (forbidden_name(c, name, CHILD(n, name_i), 0))
        return NULL;
    args = ast_for_arguments(c, CHILD(n, name_i + 1));
    if (!args)
        return NULL;
    if (TYPE(CHILD(n, name_i+2)) == RARROW) {
        returns = ast_for_expr(c, CHILD(n, name_i + 3));
        if (!returns)
            return NULL;
        name_i += 2;
    }
    if (TYPE(CHILD(n, name_i + 3)) == TYPE_COMMENT) {
        type_comment = NEW_TYPE_COMMENT(CHILD(n, name_i + 3));
        name_i += 1;
    }
    body = ast_for_suite(c, CHILD(n, name_i + 3));
    if (!body)
        return NULL;

    if (!type_comment && NCH(CHILD(n, name_i + 3)) > 1) {
        /* If the function doesn't have a type comment on the same line, check
         * if the suite has a type comment in it. */
        tc = CHILD(CHILD(n, name_i + 3), 1);

        if (TYPE(tc) == TYPE_COMMENT)
            type_comment = NEW_TYPE_COMMENT(tc);
    }

    if (is_async)
        return AsyncFunctionDef(name, args, body, decorator_seq, returns,
                                type_comment, LINENO(n0), n0->n_col_offset, c->c_arena);
    else
        return FunctionDef(name, args, body, decorator_seq, returns,
                           type_comment, LINENO(n), n->n_col_offset, c->c_arena);
}

Project: typed_ast

decode_utf8(struct compiling *c, const char **sPtr, const char *end)
{
#ifndef Py_USING_UNICODE
        Py_FatalError("decode_utf8 should not be called in this build.");
        return NULL;
#else
    const char *s, *t;
    t = s = *sPtr;
    /* while (s < end && *s != '\\') s++; */ /* inefficient for u".." */
    while (s < end && (*s & 0x80)) s++;
    *sPtr = s;
    return PyUnicode_DecodeUTF8(t, s - t, NULL);
#endif
}

Project: typed_ast

obj2ast_arguments(PyObject* obj, arguments_ty* out, PyArena* arena)
{
    PyObject* tmp = NULL;
    asdl_seq* args;
    arg_ty vararg;
    asdl_seq* kwonlyargs;
    asdl_seq* kw_defaults;
    arg_ty kwarg;
    asdl_seq* defaults;

    if (lookup_attr_id(obj, &PyId_args, &tmp) < 0) {
        return 1;
    }
    if (tmp == NULL) {
        PyErr_SetString(PyExc_TypeError, "required field \"args\" missing from arguments");
        return 1;
    }
    else {
        int res;
        Py_ssize_t len;
        Py_ssize_t i;
        if (!PyList_Check(tmp)) {
            PyErr_Format(PyExc_TypeError, "arguments field \"args\" must be a list, not a %.200s", tmp->ob_type->tp_name);
            goto failed;
        }
        len = PyList_GET_SIZE(tmp);
        args = _Ta3_asdl_seq_new(len, arena);
        if (args == NULL) goto failed;
        for (i = 0; i < len; i++) {
            arg_ty val;
            res = obj2ast_arg(PyList_GET_ITEM(tmp, i), &val, arena);
            if (res != 0) goto failed;
            if (len != PyList_GET_SIZE(tmp)) {
                PyErr_SetString(PyExc_RuntimeError, "arguments field \"args\" changed size during iteration");
                goto failed;
            }
            asdl_seq_SET(args, i, val);
        }
        Py_CLEAR(tmp);
    }
    if (lookup_attr_id(obj, &PyId_vararg, &tmp) < 0) {
        return 1;
    }
    if (tmp == NULL || tmp == Py_None) {
        Py_CLEAR(tmp);
        vararg = NULL;
    }
    else {
        int res;
        res = obj2ast_arg(tmp, &vararg, arena);
        if (res != 0) goto failed;
        Py_CLEAR(tmp);
    }
    if (lookup_attr_id(obj, &PyId_kwonlyargs, &tmp) < 0) {
        return 1;
    }
    if (tmp == NULL) {
        PyErr_SetString(PyExc_TypeError, "required field \"kwonlyargs\" missing from arguments");
        return 1;
    }
    else {
        int res;
        Py_ssize_t len;
        Py_ssize_t i;
        if (!PyList_Check(tmp)) {
            PyErr_Format(PyExc_TypeError, "arguments field \"kwonlyargs\" must be a list, not a %.200s", tmp->ob_type->tp_name);
            goto failed;
        }
        len = PyList_GET_SIZE(tmp);
        kwonlyargs = _Ta3_asdl_seq_new(len, arena);
        if (kwonlyargs == NULL) goto failed;
        for (i = 0; i < len; i++) {
            arg_ty val;
            res = obj2ast_arg(PyList_GET_ITEM(tmp, i), &val, arena);
            if (res != 0) goto failed;
            if (len != PyList_GET_SIZE(tmp)) {
                PyErr_SetString(PyExc_RuntimeError, "arguments field \"kwonlyargs\" changed size during iteration");
                goto failed;
            }
            asdl_seq_SET(kwonlyargs, i, val);
        }
        Py_CLEAR(tmp);
    }
    if (lookup_attr_id(obj, &PyId_kw_defaults, &tmp) < 0) {
        return 1;
    }
    if (tmp == NULL) {
        PyErr_SetString(PyExc_TypeError, "required field \"kw_defaults\" missing from arguments");
        return 1;
    }
    else {
        int res;
        Py_ssize_t len;
        Py_ssize_t i;
        if (!PyList_Check(tmp)) {
            PyErr_Format(PyExc_TypeError, "arguments field \"kw_defaults\" must be a list, not a %.200s", tmp->ob_type->tp_name);
            goto failed;
        }
        len = PyList_GET_SIZE(tmp);
        kw_defaults = _Ta3_asdl_seq_new(len, arena);
        if (kw_defaults == NULL) goto failed;
        for (i = 0; i < len; i++) {
            expr_ty val;
            res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &val, arena);
            if (res != 0) goto failed;
            if (len != PyList_GET_SIZE(tmp)) {
                PyErr_SetString(PyExc_RuntimeError, "arguments field \"kw_defaults\" changed size during iteration");
                goto failed;
            }
            asdl_seq_SET(kw_defaults, i, val);
        }
        Py_CLEAR(tmp);
    }
    if (lookup_attr_id(obj, &PyId_kwarg, &tmp) < 0) {
        return 1;
    }
    if (tmp == NULL || tmp == Py_None) {
        Py_CLEAR(tmp);
        kwarg = NULL;
    }
    else {
        int res;
        res = obj2ast_arg(tmp, &kwarg, arena);
        if (res != 0) goto failed;
        Py_CLEAR(tmp);
    }
    if (lookup_attr_id(obj, &PyId_defaults, &tmp) < 0) {
        return 1;
    }
    if (tmp == NULL) {
        PyErr_SetString(PyExc_TypeError, "required field \"defaults\" missing from arguments");
        return 1;
    }
    else {
        int res;
        Py_ssize_t len;
        Py_ssize_t i;
        if (!PyList_Check(tmp)) {
            PyErr_Format(PyExc_TypeError, "arguments field \"defaults\" must be a list, not a %.200s", tmp->ob_type->tp_name);
            goto failed;
        }
        len = PyList_GET_SIZE(tmp);
        defaults = _Ta3_asdl_seq_new(len, arena);
        if (defaults == NULL) goto failed;
        for (i = 0; i < len; i++) {
            expr_ty val;
            res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &val, arena);
            if (res != 0) goto failed;
            if (len != PyList_GET_SIZE(tmp)) {
                PyErr_SetString(PyExc_RuntimeError, "arguments field \"defaults\" changed size during iteration");
                goto failed;
            }
            asdl_seq_SET(defaults, i, val);
        }
        Py_CLEAR(tmp);
    }
    *out = arguments(args, vararg, kwonlyargs, kw_defaults, kwarg, defaults,
                     arena);
    return 0;
failed:
    Py_XDECREF(tmp);
    return 1;
}

Project: typed_ast

ast_for_class_bases(struct compiling *c, const node* n)
{
    /* testlist: test (',' test)* [','] */
    assert(NCH(n) > 0);
    REQ(n, testlist);
    if (NCH(n) == 1) {
        expr_ty base;
        asdl_seq *bases = asdl_seq_new(1, c->c_arena);
        if (!bases)
            return NULL;
        base = ast_for_expr(c, CHILD(n, 0));
        if (!base)
            return NULL;
        asdl_seq_SET(bases, 0, base);
        return bases;
    }

    return seq_for_testlist(c, n);
}

Project: typed_ast

ClassDef(identifier name, asdl_seq * bases, asdl_seq * keywords, asdl_seq *
         body, asdl_seq * decorator_list, int lineno, int col_offset, PyArena
         *arena)
{
    stmt_ty p;
    if (!name) {
        PyErr_SetString(PyExc_ValueError,
                        "field name is required for ClassDef");
        return NULL;
    }
    p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->kind = ClassDef_kind;
    p->v.ClassDef.name = name;
    p->v.ClassDef.bases = bases;
    p->v.ClassDef.keywords = keywords;
    p->v.ClassDef.body = body;
    p->v.ClassDef.decorator_list = decorator_list;
    p->lineno = lineno;
    p->col_offset = col_offset;
    return p;
}

Project: typed_ast
File: tokenizer.c

decoding_feof(struct tok_state *tok)
{
    if (tok->decoding_state != STATE_NORMAL) {
        return feof(tok->fp);
    } else {
        PyObject* buf = tok->decoding_buffer;
        if (buf == NULL) {
            buf = _PyObject_CallNoArg(tok->decoding_readline);
            if (buf == NULL) {
                error_ret(tok);
                return 1;
            } else {
                tok->decoding_buffer = buf;
            }
        }
        return PyObject_Length(buf) == 0;
    }
}

Project: typed_ast

Dict(asdl_seq * keys, asdl_seq * values, int lineno, int col_offset, PyArena
     *arena)
{
    expr_ty p;
    p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->kind = Dict_kind;
    p->v.Dict.keys = keys;
    p->v.Dict.values = values;
    p->lineno = lineno;
    p->col_offset = col_offset;
    return p;
}

Project: typed_ast

Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, int lineno, int
     col_offset, PyArena *arena)
{
    expr_ty p;
    if (!func) {
        PyErr_SetString(PyExc_ValueError,
                        "field func is required for Call");
        return NULL;
    }
    p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->kind = Call_kind;
    p->v.Call.func = func;
    p->v.Call.args = args;
    p->v.Call.keywords = keywords;
    p->lineno = lineno;
    p->col_offset = col_offset;
    return p;
}

Project: typed_ast

obj2ast_comprehension(PyObject* obj, comprehension_ty* out, PyArena* arena)
{
    PyObject* tmp = NULL;
    expr_ty target;
    expr_ty iter;
    asdl_seq* ifs;
    int is_async;

    if (lookup_attr_id(obj, &PyId_target, &tmp) < 0) {
        return 1;
    }
    if (tmp == NULL) {
        PyErr_SetString(PyExc_TypeError, "required field \"target\" missing from comprehension");
        return 1;
    }
    else {
        int res;
        res = obj2ast_expr(tmp, &target, arena);
        if (res != 0) goto failed;
        Py_CLEAR(tmp);
    }
    if (lookup_attr_id(obj, &PyId_iter, &tmp) < 0) {
        return 1;
    }
    if (tmp == NULL) {
        PyErr_SetString(PyExc_TypeError, "required field \"iter\" missing from comprehension");
        return 1;
    }
    else {
        int res;
        res = obj2ast_expr(tmp, &iter, arena);
        if (res != 0) goto failed;
        Py_CLEAR(tmp);
    }
    if (lookup_attr_id(obj, &PyId_ifs, &tmp) < 0) {
        return 1;
    }
    if (tmp == NULL) {
        PyErr_SetString(PyExc_TypeError, "required field \"ifs\" missing from comprehension");
        return 1;
    }
    else {
        int res;
        Py_ssize_t len;
        Py_ssize_t i;
        if (!PyList_Check(tmp)) {
            PyErr_Format(PyExc_TypeError, "comprehension field \"ifs\" must be a list, not a %.200s", tmp->ob_type->tp_name);
            goto failed;
        }
        len = PyList_GET_SIZE(tmp);
        ifs = _Ta3_asdl_seq_new(len, arena);
        if (ifs == NULL) goto failed;
        for (i = 0; i < len; i++) {
            expr_ty val;
            res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &val, arena);
            if (res != 0) goto failed;
            if (len != PyList_GET_SIZE(tmp)) {
                PyErr_SetString(PyExc_RuntimeError, "comprehension field \"ifs\" changed size during iteration");
                goto failed;
            }
            asdl_seq_SET(ifs, i, val);
        }
        Py_CLEAR(tmp);
    }
    if (lookup_attr_id(obj, &PyId_is_async, &tmp) < 0) {
        return 1;
    }
    if (tmp == NULL) {
        PyErr_SetString(PyExc_TypeError, "required field \"is_async\" missing from comprehension");
        return 1;
    }
    else {
        int res;
        res = obj2ast_int(tmp, &is_async, arena);
        if (res != 0) goto failed;
        Py_CLEAR(tmp);
    }
    *out = comprehension(target, iter, ifs, is_async, arena);
    return 0;
failed:
    Py_XDECREF(tmp);
    return 1;
}

Project: typed_ast

count_list_ifs(struct compiling *c, const node *n)
{
    int n_ifs = 0;

 count_list_iter:
    REQ(n, list_iter);
    if (TYPE(CHILD(n, 0)) == list_for)
        return n_ifs;
    n = CHILD(n, 0);
    REQ(n, list_if);
    n_ifs++;
    if (NCH(n) == 2)
        return n_ifs;
    n = CHILD(n, 2);
    goto count_list_iter;
}

Project: typed_ast

obj2ast_withitem(PyObject* obj, withitem_ty* out, PyArena* arena)
{
    PyObject* tmp = NULL;
    expr_ty context_expr;
    expr_ty optional_vars;

    if (lookup_attr_id(obj, &PyId_context_expr, &tmp) < 0) {
        return 1;
    }
    if (tmp == NULL) {
        PyErr_SetString(PyExc_TypeError, "required field \"context_expr\" missing from withitem");
        return 1;
    }
    else {
        int res;
        res = obj2ast_expr(tmp, &context_expr, arena);
        if (res != 0) goto failed;
        Py_CLEAR(tmp);
    }
    if (lookup_attr_id(obj, &PyId_optional_vars, &tmp) < 0) {
        return 1;
    }
    if (tmp == NULL || tmp == Py_None) {
        Py_CLEAR(tmp);
        optional_vars = NULL;
    }
    else {
        int res;
        res = obj2ast_expr(tmp, &optional_vars, arena);
        if (res != 0) goto failed;
        Py_CLEAR(tmp);
    }
    *out = withitem(context_expr, optional_vars, arena);
    return 0;
failed:
    Py_XDECREF(tmp);
    return 1;
}

Project: typed_ast

ast_for_with_stmt(struct compiling *c, const node *n0, bool is_async)
{
    const node * const n = is_async ? CHILD(n0, 1) : n0;
    int i, n_items, nch_minus_type, has_type_comment;
    asdl_seq *items, *body;
    string type_comment;

    if (is_async && c->c_feature_version < 5) {
        ast_error(c, n,
                "Async with statements are only supported in Python 3.5 and greater");
        return NULL;
    }

    REQ(n, with_stmt);

    has_type_comment = TYPE(CHILD(n, NCH(n) - 2)) == TYPE_COMMENT;
    nch_minus_type = NCH(n) - has_type_comment;

    n_items = (nch_minus_type - 2) / 2;
    items = _Ta3_asdl_seq_new(n_items, c->c_arena);
    if (!items)
        return NULL;
    for (i = 1; i < nch_minus_type - 2; i += 2) {
        withitem_ty item = ast_for_with_item(c, CHILD(n, i));
        if (!item)
            return NULL;
        asdl_seq_SET(items, (i - 1) / 2, item);
    }

    body = ast_for_suite(c, CHILD(n, NCH(n) - 1));
    if (!body)
        return NULL;

    if (has_type_comment)
        type_comment = NEW_TYPE_COMMENT(CHILD(n, NCH(n) - 2));
    else
        type_comment = NULL;

    if (is_async)
        return AsyncWith(items, body, type_comment, LINENO(n0), n0->n_col_offset, c->c_arena);
    else
        return With(items, body, type_comment, LINENO(n), n->n_col_offset, c->c_arena);
}

Project: typed_ast

ImportFrom(identifier module, asdl_seq * names, int level, int lineno, int
           col_offset, PyArena *arena)
{
    stmt_ty p;
    p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->kind = ImportFrom_kind;
    p->v.ImportFrom.module = module;
    p->v.ImportFrom.names = names;
    p->v.ImportFrom.level = level;
    p->lineno = lineno;
    p->col_offset = col_offset;
    return p;
}

Project: typed_ast

new_identifier(const char* n, PyArena *arena) {
    PyObject* id = PyUnicode_InternFromString(n);
    if (id != NULL)
        PyArena_AddPyObject(arena, id);
    return id;
}

Project: typed_ast

ast_error_finish(const char *filename)
{
    PyObject *type, *value, *tback, *errstr, *loc, *tmp;
    long lineno;

    assert(PyErr_Occurred());
    if (!PyErr_ExceptionMatches(PyExc_SyntaxError))
        return;

    PyErr_Fetch(&type, &value, &tback);
    errstr = PyTuple_GetItem(value, 0);
    if (!errstr)
        return;
    Py_INCREF(errstr);
    lineno = PyLong_AsLong(PyTuple_GetItem(value, 1));
    if (lineno == -1) {
        Py_DECREF(errstr);
        return;
    }
    Py_DECREF(value);

    loc = PyErr_ProgramText(filename, lineno);
    if (!loc) {
        Py_INCREF(Py_None);
        loc = Py_None;
    }
    tmp = Py_BuildValue("(zlOO)", filename, lineno, Py_None, loc);
    Py_DECREF(loc);
    if (!tmp) {
        Py_DECREF(errstr);
        return;
    }
    value = PyTuple_Pack(2, errstr, tmp);
    Py_DECREF(errstr);
    Py_DECREF(tmp);
    if (!value)
        return;
    PyErr_Restore(type, value, tback);
}

Project: typed_ast

Ta3Tokenizer_FromFile(FILE *fp, const char* enc,
                     const char *ps1, const char *ps2)
{
    struct tok_state *tok = tok_new();
    if (tok == NULL)
        return NULL;
    if ((tok->buf = (char *)PyMem_MALLOC(BUFSIZ)) == NULL) {
        Ta3Tokenizer_Free(tok);
        return NULL;
    }
    tok->cur = tok->inp = tok->buf;
    tok->end = tok->buf + BUFSIZ;
    tok->fp = fp;
    tok->prompt = ps1;
    tok->nextprompt = ps2;
    if (enc != NULL) {
        /* Must copy encoding declaration since it
           gets copied into the parse tree. */
        tok->encoding = PyMem_MALLOC(strlen(enc)+1);
        if (!tok->encoding) {
            Ta3Tokenizer_Free(tok);
            return NULL;
        }
        strcpy(tok->encoding, enc);
        tok->decoding_state = STATE_NORMAL;
    }
    return tok;
}

Project: typed_ast
File: tokenizer.c

fp_readl(char *s, int size, struct tok_state *tok)
{
    PyObject* bufobj;
    const char *buf;
    Py_ssize_t buflen;

    /* Ask for one less byte so we can terminate it */
    assert(size > 0);
    size--;

    if (tok->decoding_buffer) {
        bufobj = tok->decoding_buffer;
        Py_INCREF(bufobj);
    }
    else
    {
        bufobj = _PyObject_CallNoArg(tok->decoding_readline);
        if (bufobj == NULL)
            goto error;
    }
    if (PyUnicode_CheckExact(bufobj))
    {
        buf = PyUnicode_AsUTF8AndSize(bufobj, &buflen);
        if (buf == NULL) {
            goto error;
        }
    }
    else
    {
        buf = PyByteArray_AsString(bufobj);
        if (buf == NULL) {
            goto error;
        }
        buflen = PyByteArray_GET_SIZE(bufobj);
    }

    Py_XDECREF(tok->decoding_buffer);
    if (buflen > size) {
        /* Too many chars, the rest goes into tok->decoding_buffer */
        tok->decoding_buffer = PyByteArray_FromStringAndSize(buf+size,
                                                         buflen-size);
        if (tok->decoding_buffer == NULL)
            goto error;
        buflen = size;
    }
    else
        tok->decoding_buffer = NULL;

    memcpy(s, buf, buflen);
    s[buflen] = '\0';
    if (buflen == 0) /* EOF */
        s = NULL;
    Py_DECREF(bufobj);
    return s;

error:
    Py_XDECREF(bufobj);
    return error_ret(tok);
}

Project: typed_ast
File: tokenizer.c

fp_setreadl(struct tok_state *tok, const char* enc)
{
    PyObject *readline, *io, *stream;
    _Py_IDENTIFIER(open);
    _Py_IDENTIFIER(readline);
    int fd;
    long pos;

    fd = fileno(tok->fp);
    /* Due to buffering the file offset for fd can be different from the file
     * position of tok->fp.  If tok->fp was opened in text mode on Windows,
     * its file position counts CRLF as one char and can't be directly mapped
     * to the file offset for fd.  Instead we step back one byte and read to
     * the end of line.*/
    pos = ftell(tok->fp);
    if (pos == -1 ||
        lseek(fd, (off_t)(pos > 0 ? pos - 1 : pos), SEEK_SET) == (off_t)-1) {
        PyErr_SetFromErrnoWithFilename(PyExc_OSError, NULL);
        return 0;
    }

    io = PyImport_ImportModuleNoBlock("io");
    if (io == NULL)
        return 0;

    stream = _PyObject_CallMethodId(io, &PyId_open, "isisOOO",
                    fd, "r", -1, enc, Py_None, Py_None, Py_False);
    Py_DECREF(io);
    if (stream == NULL)
        return 0;

    readline = _PyObject_GetAttrId(stream, &PyId_readline);
    Py_DECREF(stream);
    if (readline == NULL)
        return 0;
    Py_XSETREF(tok->decoding_readline, readline);

    if (pos > 0) {
        PyObject *bufobj = _PyObject_CallNoArg(readline);
        if (bufobj == NULL)
            return 0;
        Py_DECREF(bufobj);
    }

    return 1;
}

Project: typed_ast
File: ast.c

ast_for_async_funcdef(struct compiling *c, const node *n, asdl_seq *decorator_seq)
{
    /* async_funcdef: 'async' funcdef */
    REQ(n, async_funcdef);
    REQ(CHILD(n, 0), NAME);
    assert(strcmp(STR(CHILD(n, 0)), "async") == 0);
    REQ(CHILD(n, 1), funcdef);

    return ast_for_funcdef_impl(c, n, decorator_seq,
                                true /* is_async */);
}

Project: typed_ast
File: tokenizer.c

buf_setreadl(struct tok_state *tok, const char* enc) {
    tok->enc = enc;
    return 1;
}

Project: typed_ast

ast_type_init(PyObject *self, PyObject *args, PyObject *kw)
{
    Py_ssize_t i, numfields = 0;
    int res = -1;
    PyObject *key, *value, *fields;
    if (lookup_attr_id((PyObject*)Py_TYPE(self), &PyId__fields, &fields) < 0) {
        goto cleanup;
    }
    if (fields) {
        numfields = PySequence_Size(fields);
        if (numfields == -1)
            goto cleanup;
    }

    res = 0; /* if no error occurs, this stays 0 to the end */
    if (numfields < PyTuple_GET_SIZE(args)) {
        PyErr_Format(PyExc_TypeError, "%.400s constructor takes at most "
                     "%zd positional argument%s",
                     Py_TYPE(self)->tp_name,
                     numfields, numfields == 1 ? "" : "s");
        res = -1;
        goto cleanup;
    }
    for (i = 0; i < PyTuple_GET_SIZE(args); i++) {
        /* cannot be reached when fields is NULL */
        PyObject *name = PySequence_GetItem(fields, i);
        if (!name) {
            res = -1;
            goto cleanup;
        }
        res = PyObject_SetAttr(self, name, PyTuple_GET_ITEM(args, i));
        Py_DECREF(name);
        if (res < 0)
            goto cleanup;
    }
    if (kw) {
        i = 0;  /* needed by PyDict_Next */
        while (PyDict_Next(kw, &i, &key, &value)) {
            res = PyObject_SetAttr(self, key, value);
            if (res < 0)
                goto cleanup;
        }
    }
  cleanup:
    Py_XDECREF(fields);
    return res;
}

Project: typed_ast

static int lookup_attr_id(PyObject *v, _Py_Identifier *name, PyObject **result)
{
    PyObject *oname = _PyUnicode_FromId(name); /* borrowed */
    if (!oname) {
        *result = NULL;
        return -1;
    }
    *result = PyObject_GetAttr(v, oname);
    if (*result == NULL) {
        if (!PyErr_ExceptionMatches(PyExc_AttributeError)) {
            return -1;
        }
        PyErr_Clear();
    }
    return 0;
}

Project: typed_ast

Break(int lineno, int col_offset, PyArena *arena)
{
    stmt_ty p;
    p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->kind = Break_kind;
    p->lineno = lineno;
    p->col_offset = col_offset;
    return p;
}

Project: typed_ast

ast_for_stmt(struct compiling *c, const node *n)
{
    if (TYPE(n) == stmt) {
        assert(NCH(n) == 1);
        n = CHILD(n, 0);
    }
    if (TYPE(n) == simple_stmt) {
        assert(num_stmts(n) == 1);
        n = CHILD(n, 0);
    }
    if (TYPE(n) == small_stmt) {
        n = CHILD(n, 0);
        /* small_stmt: expr_stmt | print_stmt  | del_stmt | pass_stmt
                     | flow_stmt | import_stmt | global_stmt | exec_stmt
                     | assert_stmt
        */
        switch (TYPE(n)) {
            case expr_stmt:
                return ast_for_expr_stmt(c, n);
            case print_stmt:
                return ast_for_print_stmt(c, n);
            case del_stmt:
                return ast_for_del_stmt(c, n);
            case pass_stmt:
                return Pass(LINENO(n), n->n_col_offset, c->c_arena);
            case flow_stmt:
                return ast_for_flow_stmt(c, n);
            case import_stmt:
                return ast_for_import_stmt(c, n);
            case global_stmt:
                return ast_for_global_stmt(c, n);
            case exec_stmt:
                return ast_for_exec_stmt(c, n);
            case assert_stmt:
                return ast_for_assert_stmt(c, n);
            default:
                PyErr_Format(PyExc_SystemError,
                             "unhandled small_stmt: TYPE=%d NCH=%d\n",
                             TYPE(n), NCH(n));
                return NULL;
        }
    }
    else {
        /* compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt
                        | funcdef | classdef | decorated
        */
        node *ch = CHILD(n, 0);
        REQ(n, compound_stmt);
        switch (TYPE(ch)) {
            case if_stmt:
                return ast_for_if_stmt(c, ch);
            case while_stmt:
                return ast_for_while_stmt(c, ch);
            case for_stmt:
                return ast_for_for_stmt(c, ch);
            case try_stmt:
                return ast_for_try_stmt(c, ch);
            case with_stmt:
                return ast_for_with_stmt(c, ch);
            case funcdef:
                return ast_for_funcdef(c, ch, NULL);
            case classdef:
                return ast_for_classdef(c, ch, NULL);
            case decorated:
                return ast_for_decorated(c, ch);
            default:
                PyErr_Format(PyExc_SystemError,
                             "unhandled small_stmt: TYPE=%d NCH=%d\n",
                             TYPE(n), NCH(n));
                return NULL;
        }
    }
}

Project: typed_ast
File: Python-ast.c

static int add_ast_fields(void)
{
    PyObject *empty_tuple, *d;
    if (PyType_Ready(&AST_type) < 0)
        return -1;
    d = AST_type.tp_dict;
    empty_tuple = PyTuple_New(0);
    if (!empty_tuple ||
        _PyDict_SetItemId(d, &PyId__fields, empty_tuple) < 0 ||
        _PyDict_SetItemId(d, &PyId__attributes, empty_tuple) < 0) {
        Py_XDECREF(empty_tuple);
        return -1;
    }
    Py_DECREF(empty_tuple);
    return 0;
}

Project: typed_ast

PyObject* ast2obj_expr_context(expr_context_ty o)
{
    switch(o) {
        case Load:
            Py_INCREF(Load_singleton);
            return Load_singleton;
        case Store:
            Py_INCREF(Store_singleton);
            return Store_singleton;
        case Del:
            Py_INCREF(Del_singleton);
            return Del_singleton;
        case AugLoad:
            Py_INCREF(AugLoad_singleton);
            return AugLoad_singleton;
        case AugStore:
            Py_INCREF(AugStore_singleton);
            return AugStore_singleton;
        case Param:
            Py_INCREF(Param_singleton);
            return Param_singleton;
        default:
            /* should never happen, but just in case ... */
            PyErr_Format(PyExc_SystemError, "unknown expr_context found");
            return NULL;
    }
}

Project: typed_ast

Ta3Tokenizer_FindEncoding(int fd)
{
    return Ta3Tokenizer_FindEncodingFilename(fd, NULL);
}

Project: typed_ast

Ta3Parser_ParseFile(FILE *fp, const char *filename, grammar *g, int start,
                   const char *ps1, const char *ps2,
                   perrdetail *err_ret)
{
    return Ta3Parser_ParseFileFlags(fp, filename, NULL,
                                   g, start, ps1, ps2, err_ret, 0);
}

Project: typed_ast
File: Python-ast.c

Interactive(asdl_seq * body, PyArena *arena)
{
    mod_ty p;
    p = (mod_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->kind = Interactive_kind;
    p->v.Interactive.body = body;
    return p;
}

Project: typed_ast
File: Python-ast.c

static int obj2ast_identifier(PyObject* obj, PyObject** out, PyArena* arena)
{
    if (!PyUnicode_CheckExact(obj) && obj != Py_None) {
        PyErr_SetString(PyExc_TypeError, "AST identifier must be of type str");
        return 1;
    }
    return obj2ast_object(obj, out, arena);
}

Project: typed_ast

int growable_int_array_init(growable_int_array *arr, size_t initial_size) {
    assert(initial_size > 0);
    arr->items = malloc(initial_size * sizeof(*arr->items));
    arr->size = initial_size;
    arr->num_items = 0;

    return arr->items != NULL;
}

Project: typed_ast
File: Python-ast.c

ast2obj_keyword(void* _o)
{
    keyword_ty o = (keyword_ty)_o;
    PyObject *result = NULL, *value = NULL;
    if (!o) {
        Py_RETURN_NONE;
    }

    result = PyType_GenericNew(keyword_type, NULL, NULL);
    if (!result) return NULL;
    value = ast2obj_identifier(o->arg);
    if (!value) goto failed;
    if (_PyObject_SetAttrId(result, &PyId_arg, value) == -1)
        goto failed;
    Py_DECREF(value);
    value = ast2obj_expr(o->value);
    if (!value) goto failed;
    if (_PyObject_SetAttrId(result, &PyId_value, value) == -1)
        goto failed;
    Py_DECREF(value);
    return result;
failed:
    Py_XDECREF(value);
    Py_XDECREF(result);
    return NULL;
}

Project: typed_ast
File: Python-ast.c

Suite(asdl_seq * body, PyArena *arena)
{
    mod_ty p;
    p = (mod_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->kind = Suite_kind;
    p->v.Suite.body = body;
    return p;
}

Project: typed_ast

fstring_compile_expr(const char *expr_start, const char *expr_end,
                     struct compiling *c, const node *n)

{
    PyCompilerFlags cf;
    node *mod_n;
    mod_ty mod;
    char *str;
    Py_ssize_t len;
    const char *s;
    PyObject *fstring_name;

    assert(expr_end >= expr_start);
    assert(*(expr_start-1) == '{');
    assert(*expr_end == '}' || *expr_end == '!' || *expr_end == ':');

    /* If the substring is all whitespace, it's an error.  We need to catch this
       here, and not when we call PyParser_SimpleParseStringFlagsFilename,
       because turning the expression '' in to '()' would go from being invalid
       to valid. */
    for (s = expr_start; s != expr_end; s++) {
        char c = *s;
        /* The Python parser ignores only the following whitespace
           characters (\r already is converted to \n). */
        if (!(c == ' ' || c == '\t' || c == '\n' || c == '\f')) {
            break;
        }
    }
    if (s == expr_end) {
        ast_error(c, n, "f-string: empty expression not allowed");
        return NULL;
    }

    len = expr_end - expr_start;
    /* Allocate 3 extra bytes: open paren, close paren, null byte. */
    str = PyMem_RawMalloc(len + 3);
    if (str == NULL) {
        PyErr_NoMemory();
        return NULL;
    }

    str[0] = '(';
    memcpy(str+1, expr_start, len);
    str[len+1] = ')';
    str[len+2] = 0;

    cf.cf_flags = PyCF_ONLY_AST;
    mod_n = PyParser_SimpleParseStringFlagsFilename(str, "<fstring>",
                                                    Py_eval_input, 0);
    if (!mod_n) {
        PyMem_RawFree(str);
        return NULL;
    }
    /* Reuse str to find the correct column offset. */
    str[0] = '{';
    str[len+1] = '}';
    fstring_fix_node_location(n, mod_n, str);
    fstring_name = PyUnicode_FromString("<fstring>");
    mod = string_object_to_c_ast(str, fstring_name,
                                 Py_eval_input, &cf,
                                 c->c_feature_version, c->c_arena);
    Py_DECREF(fstring_name);
    PyMem_RawFree(str);
    Ta3Node_Free(mod_n);
    if (!mod)
        return NULL;
    return mod->v.Expression.body;
}

Project: typed_ast

FormattedValue(expr_ty value, int conversion, expr_ty format_spec, int lineno,
               int col_offset, PyArena *arena)
{
    expr_ty p;
    if (!value) {
        PyErr_SetString(PyExc_ValueError,
                        "field value is required for FormattedValue");
        return NULL;
    }
    p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->kind = FormattedValue_kind;
    p->v.FormattedValue.value = value;
    p->v.FormattedValue.conversion = conversion;
    p->v.FormattedValue.format_spec = format_spec;
    p->lineno = lineno;
    p->col_offset = col_offset;
    return p;
}

Project: typed_ast

Ta3Tokenizer_Get(struct tok_state *tok, char **p_start, char **p_end)
{
    int result = tok_get(tok, p_start, p_end);
    if (tok->decoding_erred) {
        result = ERRORTOKEN;
        tok->done = E_DECODE;
    }
    return result;
}

Project: typed_ast

Ta3Parser_ParseStringFlagsFilename(const char *s, const char *filename,
                          grammar *g, int start,
                          perrdetail *err_ret, int flags)
{
    int iflags = flags;
    return Ta3Parser_ParseStringFlagsFilenameEx(s, filename, g, start,
                                               err_ret, &iflags);
}

Project: typed_ast

static int obj2ast_constant(PyObject* obj, PyObject** out, PyArena* arena)
{
    if (obj) {
        if (PyArena_AddPyObject(arena, obj) < 0) {
            *out = NULL;
            return -1;
        }
        Py_INCREF(obj);
    }
    *out = obj;
    return 0;
}

Project: typed_ast

tok_get(struct tok_state *tok, char **p_start, char **p_end)
{
    int c;
    int blankline, nonascii;

    *p_start = *p_end = NULL;
  nextline:
    tok->start = NULL;
    blankline = 0;

    /* Get indentation level */
    if (tok->atbol) {
        int col = 0;
        int altcol = 0;
        tok->atbol = 0;
        for (;;) {
            c = tok_nextc(tok);
            if (c == ' ') {
                col++, altcol++;
            }
            else if (c == '\t') {
                col = (col / tok->tabsize + 1) * tok->tabsize;
                altcol = (altcol / ALTTABSIZE + 1) * ALTTABSIZE;
            }
            else if (c == '\014')  {/* Control-L (formfeed) */
                col = altcol = 0; /* For Emacs users */
            }
            else {
                break;
            }
        }
        tok_backup(tok, c);
        if (c == '#' || c == '\n') {
            /* Lines with only whitespace and/or comments
               shouldn't affect the indentation and are
               not passed to the parser as NEWLINE tokens,
               except *totally* empty lines in interactive
               mode, which signal the end of a command group. */
            if (col == 0 && c == '\n' && tok->prompt != NULL) {
                blankline = 0; /* Let it through */
            }
            else {
                blankline = 1; /* Ignore completely */
            }
            /* We can't jump back right here since we still
               may need to skip to the end of a comment */
        }
        if (!blankline && tok->level == 0) {
            if (col == tok->indstack[tok->indent]) {
                /* No change */
                if (altcol != tok->altindstack[tok->indent]) {
                    return indenterror(tok);
                }
            }
            else if (col > tok->indstack[tok->indent]) {
                /* Indent -- always one */
                if (tok->indent+1 >= MAXINDENT) {
                    tok->done = E_TOODEEP;
                    tok->cur = tok->inp;
                    return ERRORTOKEN;
                }
                if (altcol <= tok->altindstack[tok->indent]) {
                    return indenterror(tok);
                }
                tok->pendin++;
                tok->indstack[++tok->indent] = col;
                tok->altindstack[tok->indent] = altcol;
            }
            else /* col < tok->indstack[tok->indent] */ {
                /* Dedent -- any number, must be consistent */
                while (tok->indent > 0 &&
                    col < tok->indstack[tok->indent]) {
                    tok->pendin--;
                    tok->indent--;
                }
                if (col != tok->indstack[tok->indent]) {
                    tok->done = E_DEDENT;
                    tok->cur = tok->inp;
                    return ERRORTOKEN;
                }
                if (altcol != tok->altindstack[tok->indent]) {
                    return indenterror(tok);
                }
            }
        }
    }

    tok->start = tok->cur;

    /* Return pending indents/dedents */
    if (tok->pendin != 0) {
        if (tok->pendin < 0) {
            tok->pendin++;
            return DEDENT;
        }
        else {
            tok->pendin--;
            return INDENT;
        }
    }

    if (tok->async_def
        && !blankline
        && tok->level == 0
        /* There was a NEWLINE after ASYNC DEF,
           so we're past the signature. */
        && tok->async_def_nl
        /* Current indentation level is less than where
           the async function was defined */
        && tok->async_def_indent >= tok->indent)
    {
        tok->async_def = 0;
        tok->async_def_indent = 0;
        tok->async_def_nl = 0;
    }

 again:
    tok->start = NULL;
    /* Skip spaces */
    do {
        c = tok_nextc(tok);
    } while (c == ' ' || c == '\t' || c == '\014');

    /* Set start of current token */
    tok->start = tok->cur - 1;

    /* Skip comment, unless it's a type comment */
    if (c == '#') {
        const char *prefix, *p, *type_start;

        while (c != EOF && c != '\n')
            c = tok_nextc(tok);

        p = tok->start;
        prefix = type_comment_prefix;
        while (*prefix && p < tok->cur) {
            if (*prefix == ' ') {
                while (*p == ' ' || *p == '\t')
                    p++;
            } else if (*prefix == *p) {
                p++;
            } else {
                break;
            }

            prefix++;
        }

        /* This is a type comment if we matched all of type_comment_prefix. */
        if (!*prefix) {
            int is_type_ignore = 1;
            tok_backup(tok, c);  /* don't eat the newline or EOF */

            type_start = p;

            is_type_ignore = tok->cur >= p + 6 && memcmp(p, "ignore", 6) == 0;
            p += 6;
            while (is_type_ignore && p < tok->cur) {
              if (*p == '#')
                  break;
              is_type_ignore = is_type_ignore && (*p == ' ' || *p == '\t');
              p++;
            }

            if (is_type_ignore) {
                /* If this type ignore is the only thing on the line, consume the newline also. */
                if (blankline) {
                    tok_nextc(tok);
                    tok->atbol = 1;
                }
                return TYPE_IGNORE;
            } else {
                *p_start = (char *) type_start;  /* after type_comment_prefix */
                *p_end = tok->cur;
                return TYPE_COMMENT;
            }
        }
    }

    /* Check for EOF and errors now */
    if (c == EOF) {
        return tok->done == E_EOF ? ENDMARKER : ERRORTOKEN;
    }

    /* Identifier (most frequent token!) */
    nonascii = 0;
    if (is_potential_identifier_start(c)) {
        /* Process the various legal combinations of b"", r"", u"", and f"". */
        int saw_b = 0, saw_r = 0, saw_u = 0, saw_f = 0;
        while (1) {
            if (!(saw_b || saw_u || saw_f) && (c == 'b' || c == 'B'))
                saw_b = 1;
            /* Since this is a backwards compatibility support literal we don't
               want to support it in arbitrary order like byte literals. */
            else if (!(saw_b || saw_u || saw_r || saw_f)
                     && (c == 'u'|| c == 'U')) {
                saw_u = 1;
            }
            /* ur"" and ru"" are not supported */
            else if (!(saw_r || saw_u) && (c == 'r' || c == 'R')) {
                saw_r = 1;
            }
            else if (!(saw_f || saw_b || saw_u) && (c == 'f' || c == 'F')) {
                saw_f = 1;
            }
            else {
                break;
            }
            c = tok_nextc(tok);
            if (c == '"' || c == '\'') {
                goto letter_quote;
            }
        }
        while (is_potential_identifier_char(c)) {
            if (c >= 128) {
                nonascii = 1;
            }
            c = tok_nextc(tok);
        }
        tok_backup(tok, c);
        if (nonascii && !verify_identifier(tok)) {
            return ERRORTOKEN;
        }
        *p_start = tok->start;
        *p_end = tok->cur;

        /* async/await parsing block. */
        if (tok->cur - tok->start == 5) {
            /* Current token length is 5. */
            if (tok->async_always || tok->async_def) {
                /* We're inside an 'async def' function. */
                if (memcmp(tok->start, "async", 5) == 0) {
                    return ASYNC;
                }
                if (memcmp(tok->start, "await", 5) == 0) {
                    return AWAIT;
                }
            }
            else if (memcmp(tok->start, "async", 5) == 0) {
                /* The current token is 'async'.
                   Look ahead one token.*/

                struct tok_state ahead_tok;
                char *ahead_tok_start = NULL, *ahead_tok_end = NULL;
                int ahead_tok_kind;

                memcpy(&ahead_tok, tok, sizeof(ahead_tok));
                ahead_tok_kind = tok_get(&ahead_tok, &ahead_tok_start,
                                         &ahead_tok_end);

                if (ahead_tok_kind == NAME
                    && ahead_tok.cur - ahead_tok.start == 3
                    && memcmp(ahead_tok.start, "def", 3) == 0)
                {
                    /* The next token is going to be 'def', so instead of
                       returning 'async' NAME token, we return ASYNC. */
                    tok->async_def_indent = tok->indent;
                    tok->async_def = 1;
                    return ASYNC;
                }
            }
        }

        return NAME;
    }

    /* Newline */
    if (c == '\n') {
        tok->atbol = 1;
        if (blankline || tok->level > 0) {
            goto nextline;
        }
        *p_start = tok->start;
        *p_end = tok->cur - 1; /* Leave '\n' out of the string */
        tok->cont_line = 0;
        if (tok->async_def) {
            /* We're somewhere inside an 'async def' function, and
               we've encountered a NEWLINE after its signature. */
            tok->async_def_nl = 1;
        }
        return NEWLINE;
    }

    /* Period or number starting with period? */
    if (c == '.') {
        c = tok_nextc(tok);
        if (isdigit(c)) {
            goto fraction;
        } else if (c == '.') {
            c = tok_nextc(tok);
            if (c == '.') {
                *p_start = tok->start;
                *p_end = tok->cur;
                return ELLIPSIS;
            }
            else {
                tok_backup(tok, c);
            }
            tok_backup(tok, '.');
        }
        else {
            tok_backup(tok, c);
        }
        *p_start = tok->start;
        *p_end = tok->cur;
        return DOT;
    }

    /* Number */
    if (isdigit(c)) {
        if (c == '0') {
            /* Hex, octal or binary -- maybe. */
            c = tok_nextc(tok);
            if (c == 'x' || c == 'X') {
                /* Hex */
                c = tok_nextc(tok);
                do {
                    if (c == '_') {
                        c = tok_nextc(tok);
                    }
                    if (!isxdigit(c)) {
                        tok->done = E_TOKEN;
                        tok_backup(tok, c);
                        return ERRORTOKEN;
                    }
                    do {
                        c = tok_nextc(tok);
                    } while (isxdigit(c));
                } while (c == '_');
            }
            else if (c == 'o' || c == 'O') {
                /* Octal */
                c = tok_nextc(tok);
                do {
                    if (c == '_') {
                        c = tok_nextc(tok);
                    }
                    if (c < '0' || c >= '8') {
                        tok->done = E_TOKEN;
                        tok_backup(tok, c);
                        return ERRORTOKEN;
                    }
                    do {
                        c = tok_nextc(tok);
                    } while ('0' <= c && c < '8');
                } while (c == '_');
            }
            else if (c == 'b' || c == 'B') {
                /* Binary */
                c = tok_nextc(tok);
                do {
                    if (c == '_') {
                        c = tok_nextc(tok);
                    }
                    if (c != '0' && c != '1') {
                        tok->done = E_TOKEN;
                        tok_backup(tok, c);
                        return ERRORTOKEN;
                    }
                    do {
                        c = tok_nextc(tok);
                    } while (c == '0' || c == '1');
                } while (c == '_');
            }
            else {
                int nonzero = 0;
                /* maybe old-style octal; c is first char of it */
                /* in any case, allow '0' as a literal */
                while (1) {
                    if (c == '_') {
                        c = tok_nextc(tok);
                        if (!isdigit(c)) {
                            tok->done = E_TOKEN;
                            tok_backup(tok, c);
                            return ERRORTOKEN;
                        }
                    }
                    if (c != '0') {
                        break;
                    }
                    c = tok_nextc(tok);
                }
                if (isdigit(c)) {
                    nonzero = 1;
                    c = tok_decimal_tail(tok);
                    if (c == 0) {
                        return ERRORTOKEN;
                    }
                }
                if (c == '.') {
                    c = tok_nextc(tok);
                    goto fraction;
                }
                else if (c == 'e' || c == 'E') {
                    goto exponent;
                }
                else if (c == 'j' || c == 'J') {
                    goto imaginary;
                }
                else if (nonzero) {
                    /* Old-style octal: now disallowed. */
                    tok->done = E_TOKEN;
                    tok_backup(tok, c);
                    return ERRORTOKEN;
                }
            }
        }
        else {
            /* Decimal */
            c = tok_decimal_tail(tok);
            if (c == 0) {
                return ERRORTOKEN;
            }
            {
                /* Accept floating point numbers. */
                if (c == '.') {
                    c = tok_nextc(tok);
        fraction:
                    /* Fraction */
                    if (isdigit(c)) {
                        c = tok_decimal_tail(tok);
                        if (c == 0) {
                            return ERRORTOKEN;
                        }
                    }
                }
                if (c == 'e' || c == 'E') {
                    int e;
                  exponent:
                    e = c;
                    /* Exponent part */
                    c = tok_nextc(tok);
                    if (c == '+' || c == '-') {
                        c = tok_nextc(tok);
                        if (!isdigit(c)) {
                            tok->done = E_TOKEN;
                            tok_backup(tok, c);
                            return ERRORTOKEN;
                        }
                    } else if (!isdigit(c)) {
                        tok_backup(tok, c);
                        tok_backup(tok, e);
                        *p_start = tok->start;
                        *p_end = tok->cur;
                        return NUMBER;
                    }
                    c = tok_decimal_tail(tok);
                    if (c == 0) {
                        return ERRORTOKEN;
                    }
                }
                if (c == 'j' || c == 'J') {
                    /* Imaginary part */
        imaginary:
                    c = tok_nextc(tok);
                }
            }
        }
        tok_backup(tok, c);
        *p_start = tok->start;
        *p_end = tok->cur;
        return NUMBER;
    }

  letter_quote:
    /* String */
    if (c == '\'' || c == '"') {
        int quote = c;
        int quote_size = 1;             /* 1 or 3 */
        int end_quote_size = 0;

        /* Find the quote size and start of string */
        c = tok_nextc(tok);
        if (c == quote) {
            c = tok_nextc(tok);
            if (c == quote) {
                quote_size = 3;
            }
            else {
                end_quote_size = 1;     /* empty string found */
            }
        }
        if (c != quote) {
            tok_backup(tok, c);
        }

        /* Get rest of string */
        while (end_quote_size != quote_size) {
            c = tok_nextc(tok);
            if (c == EOF) {
                if (quote_size == 3) {
                    tok->done = E_EOFS;
                }
                else {
                    tok->done = E_EOLS;
                }
                tok->cur = tok->inp;
                return ERRORTOKEN;
            }
            if (quote_size == 1 && c == '\n') {
                tok->done = E_EOLS;
                tok->cur = tok->inp;
                return ERRORTOKEN;
            }
            if (c == quote) {
                end_quote_size += 1;
            }
            else {
                end_quote_size = 0;
                if (c == '\\') {
                    tok_nextc(tok);  /* skip escaped char */
                }
            }
        }

        *p_start = tok->start;
        *p_end = tok->cur;
        return STRING;
    }

    /* Line continuation */
    if (c == '\\') {
        c = tok_nextc(tok);
        if (c != '\n') {
            tok->done = E_LINECONT;
            tok->cur = tok->inp;
            return ERRORTOKEN;
        }
        tok->cont_line = 1;
        goto again; /* Read next line */
    }

    /* Check for two-character token */
    {
        int c2 = tok_nextc(tok);
        int token = Ta3Token_TwoChars(c, c2);
        if (token != OP) {
            int c3 = tok_nextc(tok);
            int token3 = Ta3Token_ThreeChars(c, c2, c3);
            if (token3 != OP) {
                token = token3;
            }
            else {
                tok_backup(tok, c3);
            }
            *p_start = tok->start;
            *p_end = tok->cur;
            return token;
        }
        tok_backup(tok, c2);
    }

    /* Keep track of parentheses nesting level */
    switch (c) {
    case '(':
    case '[':
    case '{':
        tok->level++;
        break;
    case ')':
    case ']':
    case '}':
        tok->level--;
        break;
    }

    /* Punctuation character */
    *p_start = tok->start;
    *p_end = tok->cur;
    return Ta3Token_OneChar(c);
}

Project: typed_ast

ast_for_decorated(struct compiling *c, const node *n)
{
    /* decorated: decorators (classdef | funcdef) */
    stmt_ty thing = NULL;
    asdl_seq *decorator_seq = NULL;

    REQ(n, decorated);

    decorator_seq = ast_for_decorators(c, CHILD(n, 0));
    if (!decorator_seq)
      return NULL;

    assert(TYPE(CHILD(n, 1)) == funcdef ||
           TYPE(CHILD(n, 1)) == classdef);

    if (TYPE(CHILD(n, 1)) == funcdef) {
      thing = ast_for_funcdef(c, CHILD(n, 1), decorator_seq);
    } else if (TYPE(CHILD(n, 1)) == classdef) {
      thing = ast_for_classdef(c, CHILD(n, 1), decorator_seq);
    }
    /* we count the decorators in when talking about the class' or
       function's line number */
    if (thing) {
        thing->lineno = LINENO(n);
        thing->col_offset = n->n_col_offset;
    }
    return thing;
}

Project: typed_ast

Ta3Parser_ParseFileObject(FILE *fp, PyObject *filename,
                         const char *enc, grammar *g, int start,
                         const char *ps1, const char *ps2,
                         perrdetail *err_ret, int *flags)
{
    struct tok_state *tok;

    if (initerr(err_ret, filename) < 0)
        return NULL;

    if ((tok = Ta3Tokenizer_FromFile(fp, enc, ps1, ps2)) == NULL) {
        err_ret->error = E_NOMEM;
        return NULL;
    }
#ifndef PGEN
    Py_INCREF(err_ret->filename);
    tok->filename = err_ret->filename;
#endif
    return parsetok(tok, g, start, err_ret, flags);
}

Project: typed_ast
File: Python-ast.c

withitem(expr_ty context_expr, expr_ty optional_vars, PyArena *arena)
{
    withitem_ty p;
    if (!context_expr) {
        PyErr_SetString(PyExc_ValueError,
                        "field context_expr is required for withitem");
        return NULL;
    }
    p = (withitem_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->context_expr = context_expr;
    p->optional_vars = optional_vars;
    return p;
}

Project: typed_ast

obj2ast_excepthandler(PyObject* obj, excepthandler_ty* out, PyArena* arena)
{
    int isinstance;

    PyObject *tmp = NULL;
    int lineno;
    int col_offset;

    if (obj == Py_None) {
        *out = NULL;
        return 0;
    }
    if (lookup_attr_id(obj, &PyId_lineno, &tmp) < 0) {
        return 1;
    }
    if (tmp == NULL) {
        PyErr_SetString(PyExc_TypeError, "required field \"lineno\" missing from excepthandler");
        return 1;
    }
    else {
        int res;
        res = obj2ast_int(tmp, &lineno, arena);
        if (res != 0) goto failed;
        Py_CLEAR(tmp);
    }
    if (lookup_attr_id(obj, &PyId_col_offset, &tmp) < 0) {
        return 1;
    }
    if (tmp == NULL) {
        PyErr_SetString(PyExc_TypeError, "required field \"col_offset\" missing from excepthandler");
        return 1;
    }
    else {
        int res;
        res = obj2ast_int(tmp, &col_offset, arena);
        if (res != 0) goto failed;
        Py_CLEAR(tmp);
    }
    isinstance = PyObject_IsInstance(obj, (PyObject*)ExceptHandler_type);
    if (isinstance == -1) {
        return 1;
    }
    if (isinstance) {
        expr_ty type;
        identifier name;
        asdl_seq* body;

        if (lookup_attr_id(obj, &PyId_type, &tmp) < 0) {
            return 1;
        }
        if (tmp == NULL || tmp == Py_None) {
            Py_CLEAR(tmp);
            type = NULL;
        }
        else {
            int res;
            res = obj2ast_expr(tmp, &type, arena);
            if (res != 0) goto failed;
            Py_CLEAR(tmp);
        }
        if (lookup_attr_id(obj, &PyId_name, &tmp) < 0) {
            return 1;
        }
        if (tmp == NULL || tmp == Py_None) {
            Py_CLEAR(tmp);
            name = NULL;
        }
        else {
            int res;
            res = obj2ast_identifier(tmp, &name, arena);
            if (res != 0) goto failed;
            Py_CLEAR(tmp);
        }
        if (lookup_attr_id(obj, &PyId_body, &tmp) < 0) {
            return 1;
        }
        if (tmp == NULL) {
            PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from ExceptHandler");
            return 1;
        }
        else {
            int res;
            Py_ssize_t len;
            Py_ssize_t i;
            if (!PyList_Check(tmp)) {
                PyErr_Format(PyExc_TypeError, "ExceptHandler field \"body\" must be a list, not a %.200s", tmp->ob_type->tp_name);
                goto failed;
            }
            len = PyList_GET_SIZE(tmp);
            body = _Ta3_asdl_seq_new(len, arena);
            if (body == NULL) goto failed;
            for (i = 0; i < len; i++) {
                stmt_ty val;
                res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &val, arena);
                if (res != 0) goto failed;
                if (len != PyList_GET_SIZE(tmp)) {
                    PyErr_SetString(PyExc_RuntimeError, "ExceptHandler field \"body\" changed size during iteration");
                    goto failed;
                }
                asdl_seq_SET(body, i, val);
            }
            Py_CLEAR(tmp);
        }
        *out = ExceptHandler(type, name, body, lineno, col_offset, arena);
        if (*out == NULL) goto failed;
        return 0;
    }

    PyErr_Format(PyExc_TypeError, "expected some sort of excepthandler, but got %R", obj);
    failed:
    Py_XDECREF(tmp);
    return 1;
}

Project: typed_ast

obj2ast_type_ignore(PyObject* obj, type_ignore_ty* out, PyArena* arena)
{
    int isinstance;

    PyObject *tmp = NULL;

    if (obj == Py_None) {
        *out = NULL;
        return 0;
    }
    isinstance = PyObject_IsInstance(obj, (PyObject*)TypeIgnore_type);
    if (isinstance == -1) {
        return 1;
    }
    if (isinstance) {
        int lineno;

        if (lookup_attr_id(obj, &PyId_lineno, &tmp) < 0) {
            return 1;
        }
        if (tmp == NULL) {
            PyErr_SetString(PyExc_TypeError, "required field \"lineno\" missing from TypeIgnore");
            return 1;
        }
        else {
            int res;
            res = obj2ast_int(tmp, &lineno, arena);
            if (res != 0) goto failed;
            Py_CLEAR(tmp);
        }
        *out = TypeIgnore(lineno, arena);
        if (*out == NULL) goto failed;
        return 0;
    }

    PyErr_Format(PyExc_TypeError, "expected some sort of type_ignore, but got %R", obj);
    failed:
    Py_XDECREF(tmp);
    return 1;
}

Project: typed_ast

While(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, int
      col_offset, PyArena *arena)
{
    stmt_ty p;
    if (!test) {
        PyErr_SetString(PyExc_ValueError,
                        "field test is required for While");
        return NULL;
    }
    p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->kind = While_kind;
    p->v.While.test = test;
    p->v.While.body = body;
    p->v.While.orelse = orelse;
    p->lineno = lineno;
    p->col_offset = col_offset;
    return p;
}

Project: typed_ast
File: Python-ast.c

static PyObject* ast2obj_list(asdl_seq *seq, PyObject* (*func)(void*))
{
    Py_ssize_t i, n = asdl_seq_LEN(seq);
    PyObject *result = PyList_New(n);
    PyObject *value;
    if (!result)
        return NULL;
    for (i = 0; i < n; i++) {
        value = func(asdl_seq_GET(seq, i));
        if (!value) {
            Py_DECREF(result);
            return NULL;
        }
        PyList_SET_ITEM(result, i, value);
    }
    return result;
}

Project: typed_ast

int Ta3AST_Check(PyObject* obj)
{
    if (!init_types())
        return -1;
    return PyObject_IsInstance(obj, (PyObject*)&AST_type);
}