GitHub PullRequestReviewCommentEvent at 2014-08-15 20:09:18
Repository: astropy
Actor: amras1
Payload: {"action":"created","comment":{"url":"https://api.github.com/repos/astropy/astropy/pulls/comments/16312892","id":16312892,"diff_hunk":"@@ -0,0 +1,857 @@\n+// Licensed under a 3-clause BSD style license - see LICENSE.rst\n+\n+#include \"tokenizer.h\"\n+\n+tokenizer_t *create_tokenizer(char delimiter, char comment, char quotechar,\n+                              int fill_extra_cols, int strip_whitespace_lines,\n+                              int strip_whitespace_fields, int use_fast_converter)\n+{\n+    // Create the tokenizer in memory\n+    tokenizer_t *tokenizer = (tokenizer_t *) malloc(sizeof(tokenizer_t));\n+\n+    // Initialize the tokenizer fields\n+    tokenizer->source = 0;\n+    tokenizer->source_len = 0;\n+    tokenizer->source_pos = 0;\n+    tokenizer->delimiter = delimiter;\n+    tokenizer->comment = comment;\n+    tokenizer->quotechar = quotechar;\n+    tokenizer->output_cols = 0;\n+    tokenizer->col_ptrs = 0;\n+    tokenizer->output_len = 0;\n+    tokenizer->num_cols = 0;\n+    tokenizer->num_rows = 0;\n+    tokenizer->fill_extra_cols = fill_extra_cols;\n+    tokenizer->state = START_LINE;\n+    tokenizer->code = NO_ERROR;\n+    tokenizer->iter_col = 0;\n+    tokenizer->curr_pos = 0;\n+    tokenizer->strip_whitespace_lines = strip_whitespace_lines;\n+    tokenizer->strip_whitespace_fields = strip_whitespace_fields;\n+    tokenizer->use_fast_converter = use_fast_converter;\n+\n+    // This is a bit of a hack -- buf holds an empty string to represent\n+    // empty field values\n+    tokenizer->buf = calloc(2, sizeof(char));\n+\n+    return tokenizer;\n+}\n+\n+void delete_data(tokenizer_t *tokenizer)\n+{\n+    // Don't free tokenizer->source because it points to part of\n+    // an already freed Python object\n+    int i;\n+\n+    if (tokenizer->output_cols)\n+\tfor (i = 0; i < tokenizer->num_cols; ++i)\n+\t    free(tokenizer->output_cols[i]);\n+    \n+    free(tokenizer->output_cols);\n+    free(tokenizer->col_ptrs);\n+    free(tokenizer->output_len);\n+\n+    // Set pointers to 0 so we don't use freed memory when reading over again\n+    tokenizer->output_cols = 0;\n+    tokenizer->col_ptrs = 0;\n+    tokenizer->output_len = 0;\n+}\n+\n+void delete_tokenizer(tokenizer_t *tokenizer)\n+{\n+    delete_data(tokenizer);\n+    free(tokenizer->buf);\n+    free(tokenizer);\n+}\n+\n+void resize_col(tokenizer_t *self, int index)\n+{\n+    // Temporarily store the position in output_cols[index] to\n+    // which col_ptrs[index] points\n+    int diff = self->col_ptrs[index] - self->output_cols[index];\n+\n+    // Double the size of the column string\n+    self->output_cols[index] = (char *) realloc(self->output_cols[index], 2 * \n+                                                self->output_len[index] * sizeof(char));\n+\n+    // Set the second (newly allocated) half of the column string to all zeros\n+    memset(self->output_cols[index] + self->output_len[index] * sizeof(char), 0,\n+           self->output_len[index] * sizeof(char));\n+\n+    self->output_len[index] *= 2;\n+    // realloc() might move the address in memory, so we have to move\n+    // col_ptrs[index] to an offset of the new address\n+    self->col_ptrs[index] = self->output_cols[index] + diff;\n+}\n+\n+/*\n+  Resize the column string if necessary and then append c to the\n+  end of the column string, incrementing the column position pointer.\n+*/\n+\n+#define PUSH(c)                                                 \\\n+    if (self->col_ptrs[col] - self->output_cols[col] >=         \\\n+        self->output_len[col])                                  \\\n+    {                                                           \\\n+        resize_col(self, col);                                  \\\n+    }                                                           \\\n+    *self->col_ptrs[col]++ = c;\n+\n+/* Set the state to START_FIELD and begin with the assumption that\n+   the field is entirely whitespace in order to handle the possibility\n+   that the comment character is found before any non-whitespace even\n+   if whitespace stripping is disable.\n+*/\n+\n+#define BEGIN_FIELD()                           \\\n+    self->state = START_FIELD;                  \\\n+    whitespace = 1;\n+\n+/*\n+  First, backtrack to eliminate trailing whitespace if strip_whitespace_fields\n+  is true. If the field is empty, push '\\x01' as a marker.\n+  Append a null byte to the end of the column string as a field delimiting marker.\n+  Increment the variable col if we are tokenizing data.\n+*/\n+\n+#define END_FIELD()\t\t\t\t\t\t\t\\\n+    if (self->strip_whitespace_fields)                                  \\\n+    {                                                                   \\\n+        --self->col_ptrs[col];                                          \\\n+        while (*self->col_ptrs[col] == ' ' || *self->col_ptrs[col] == '\\t') \\\n+        {                                                               \\\n+            *self->col_ptrs[col]-- = '\\x00';                            \\\n+        }                                                               \\\n+        ++self->col_ptrs[col];                                          \\\n+    }                                                                   \\\n+    if (self->col_ptrs[col] == self->output_cols[col] ||                \\\n+        self->col_ptrs[col][-1] == '\\x00')                              \\\n+    {                                                                   \\\n+        PUSH('\\x01');                                                   \\\n+    }                                                                   \\\n+    PUSH('\\x00');                                                       \\\n+    if (!header)                                                        \\\n+        ++col;\n+\n+/*\n+  If we are tokenizing the header, end after the first line.\n+  Handle the possibility of insufficient columns appropriately;\n+  if fill_extra_cols=1, then append empty fields, but otherwise\n+  return an error. Increment our row count and possibly end if\n+  all the necessary rows have already been parsed.\n+*/\n+\n+#define END_LINE()                              \\\n+    if (header)                                 \\\n+    {                                           \\\n+        ++self->source_pos;                     \\\n+        RETURN(NO_ERROR);                       \\\n+    }                                           \\\n+    else if (self->fill_extra_cols)             \\\n+\twhile (col < self->num_cols)            \\\n+\t{                                       \\\n+            PUSH('\\x01');                       \\\n+\t    END_FIELD();                        \\\n+\t}                                       \\\n+    else if (col < self->num_cols)              \\\n+\tRETURN(NOT_ENOUGH_COLS);                \\\n+    ++self->num_rows;                           \\\n+    old_state = START_LINE;                     \\\n+    if (end != -1 && self->num_rows == end)     \\\n+    {                                           \\\n+        ++self->source_pos;                     \\\n+        RETURN(NO_ERROR);                       \\\n+    }\n+\n+// Set the error code to c for later retrieval and return c\n+#define RETURN(c)                               \\\n+    do {                                        \\\n+        self->code = c;                         \\\n+        return c;                               \\\n+    } while (0)\n+\n+#define HANDLE_CR() old_state = self->state; self->state = CARRIAGE_RETURN\n+\n+int skip_lines(tokenizer_t *self, int offset, int header)\n+{\n+    int signif_chars = 0;\n+    int comment = 0;\n+    int i = 0;\n+    char c;\n+    \n+    while (i < offset)\n+    {\n+\tif (self->source_pos >= self->source_len)\n+        {\n+            if (header)\n+                RETURN(INVALID_LINE); // header line is required\n+            else\n+                RETURN(NO_ERROR); // no data in input\n+        }\n+\n+        c = self->source[self->source_pos];\n+\n+        if (c == '\\r' || c == '\\n')\n+        {\n+            if (c == '\\r' && self->source_pos < self->source_len - 1 &&\n+                self->source[self->source_pos + 1] == '\\n')\n+            {\n+                ++self->source_pos; // skip \\n in \\r\\n\n+            }\n+            if (!comment && signif_chars > 0)\n+                ++i;\n+            // Start by assuming a line is empty and non-commented\n+            signif_chars = 0;\n+            comment = 0;\n+        }\n+\n+\telse if ((c != ' ' && c != '\\t') || !self->strip_whitespace_lines || header)\n+\t{ \n+            // comment line\n+            if (!signif_chars && self->comment != 0 && c == self->comment)\n+                comment = 1;\n+\n+            // significant character encountered; during header\n+            // tokenization, we count whitespace unlike data tokenization\n+            // (see #2654)\n+            ++signif_chars;\n+\t}\n+\n+        ++self->source_pos;\n+    }\n+\n+    RETURN(NO_ERROR);\n+}\n+\n+int tokenize(tokenizer_t *self, int end, int header, int num_cols)\n+{\n+    delete_data(self); // clear old reading data\n+    char c; // input character\n+    int col = 0; // current column ignoring possibly excluded columns\n+    int old_state = START_LINE; // last state the tokenizer was in before CR mode\n+    int parse_newline = 0; // explicit flag to treat current char as a newline\n+    self->num_rows = 0;\n+    int i = 0;\n+    int whitespace = 1;\n+\n+    if (header)\n+        self->num_cols = 1; // store header output in one column\n+    else\n+        self->num_cols = num_cols;\n+\n+    // Allocate memory for structures used during tokenization\n+    self->output_cols = (char **) malloc(self->num_cols * sizeof(char *));\n+    self->col_ptrs = (char **) malloc(self->num_cols * sizeof(char *));\n+    self->output_len = (int *) malloc(self->num_cols * sizeof(int));\n+\t\n+    for (i = 0; i < self->num_cols; ++i)\n+    {\n+        self->output_cols[i] = (char *) calloc(1, INITIAL_COL_SIZE *\n+                                               sizeof(char));\n+        // Make each col_ptrs pointer point to the beginning of the\n+        // column string\n+        self->col_ptrs[i] = self->output_cols[i];\n+        self->output_len[i] = INITIAL_COL_SIZE;\n+    }\n+\n+    if (end == 0)\n+        RETURN(NO_ERROR); // don't read if end == 0\n+\n+    self->state = START_LINE;\n+\n+    // Loop until all of self->source has been read\n+    while (self->source_pos < self->source_len + 1)\n+    {\n+        if (self->source_pos == self->source_len || parse_newline)\n+            c = '\\n';\n+        else\n+            c = self->source[self->source_pos];\n+\n+        parse_newline = 0;\n+\n+        switch (self->state)\n+        {\n+        case START_LINE:\n+            if (c == '\\n')\n+                break;\n+            else if (c == '\\r')\n+            {\n+                HANDLE_CR();\n+                break;\n+            }\n+            else if ((c == ' ' || c == '\\t') && self->strip_whitespace_lines)\n+                break;\n+            else if (self->comment != 0 && c == self->comment)\n+            {\n+                // comment line; ignore\n+                self->state = COMMENT;\n+                break;\n+            }\n+            // initialize variables for the beginning of line parsing\n+            col = 0;\n+            BEGIN_FIELD();\n+            // parse in mode START_FIELD\n+\t    \n+        case START_FIELD:\n+            // strip whitespace before field begins\n+            if ((c == ' ' || c == '\\t') && self->strip_whitespace_fields)\n+                break;\n+            else if (!self->strip_whitespace_lines && self->comment != 0 &&\n+                     c == self->comment)\n+            {\n+                // comment line, not caught earlier because of no stripping\n+                self->state = COMMENT;\n+                break;\n+            }\n+            else if (c == self->delimiter) // field ends before it begins\n+            {\n+                END_FIELD();\n+                BEGIN_FIELD();\n+                break;\n+            }\n+            else if (c == '\\r')\n+            {\n+                HANDLE_CR();\n+                break;\n+            }\n+            else if (c == '\\n')\n+            {\n+                if (self->strip_whitespace_lines)\n+                {\n+                    // Move on if the delimiter is whitespace, e.g.\n+                    // '1 2 3   '->['1','2','3']\n+                    if (self->delimiter == ' ' || self->delimiter == '\\t')\n+                        ;\n+                    // Register an empty field if non-whitespace delimiter,\n+                    // e.g. '1,2, '->['1','2','']\n+                    else\n+                    {\n+                        END_FIELD();\n+                    }                            \n+                }\n+\n+                else if (!self->strip_whitespace_lines)\n+                {\n+                    // In this case we don't want to left-strip the field,\n+                    // so we backtrack\n+                    int tmp = self->source_pos;\n+                    --self->source_pos;\n+\n+                    while (self->source_pos >= 0 &&\n+                           self->source[self->source_pos] != self->delimiter\n+                           && self->source[self->source_pos] != '\\n'\n+                           && self->source[self->source_pos] != '\\r')\n+                    {\n+                        --self->source_pos;\n+                    }\n+\n+                    // backtracked to line beginning\n+                    if (self->source_pos == -1 || self->source[self->source_pos] == '\\n'\n+                        || self->source[self->source_pos] == '\\r')\n+                    {\n+                        self->source_pos = tmp;\n+                    }\n+                    else\n+                    {\n+                        ++self->source_pos;\n+\n+                        if (self->source_pos == tmp)\n+                            // no whitespace, just an empty field\n+                            ;                                \n+\n+                        else\n+                            while (self->source_pos < tmp)\n+                            {\n+                                // append whitespace characters\n+                                PUSH(self->source[self->source_pos]);\n+                                ++self->source_pos;\n+                            }\n+\n+                        END_FIELD(); // whitespace counts as a field\n+                    }\n+                }\n+                        \n+                END_LINE();\n+                self->state = START_LINE;\n+                break;\n+            }\n+            else if (c == self->quotechar) // start parsing quoted field\n+            {\n+                self->state = START_QUOTED_FIELD;\n+                break;\n+            }\n+            else\n+            {\n+                if (col >= self->num_cols)\n+                    RETURN(TOO_MANY_COLS);\n+                // Valid field character, parse again in FIELD mode\n+                self->state = FIELD;\n+            }\n+\n+        case FIELD:\n+            if (self->comment != 0 && c == self->comment && whitespace && col == 0)\n+            {\n+                // No whitespace stripping, but the comment char is found\n+                // before any data, e.g. '  # a b c'\n+                self->state = COMMENT;\n+            }\n+            else if (c == self->delimiter)\n+            {\n+                // End of field, look for new field\n+                END_FIELD();\n+                BEGIN_FIELD();\n+            }\n+            else if (c == '\\r')\n+            {\n+                HANDLE_CR();\n+            }\n+            else if (c == '\\n')\n+            {\n+                // Line ending, stop parsing both field and line\n+                END_FIELD();\n+                END_LINE();\n+                self->state = START_LINE;\n+            }\n+            else\n+            {\n+                if (c != ' ' && c != '\\t')\n+                    whitespace = 0; // field is not all whitespace\n+                PUSH(c);\n+            }\n+            break;\n+\t\t\n+        case START_QUOTED_FIELD:\n+            if ((c == ' ' || c == '\\t') && self->strip_whitespace_fields)\n+            {\n+                // ignore initial whitespace\n+                break;\n+            }\n+            else if (c == self->quotechar) // empty quotes\n+            {\n+                END_FIELD();\n+                break;\n+            }\n+            else\n+            {\n+                // Valid field character, parse again in QUOTED_FIELD mode\n+                self->state = QUOTED_FIELD;\n+            }\n+\n+        case QUOTED_FIELD_NEWLINE:\n+            if (self->state == QUOTED_FIELD)\n+                ; // fall through\n+            // Ignore initial whitespace if strip_whitespace_lines and\n+            // newlines regardless\n+            else if (((c == ' ' || c == '\\t') && self->strip_whitespace_lines)\n+                     || c == '\\n')\n+                break;\n+            else if (c == '\\r')\n+            {\n+                HANDLE_CR();\n+                break;\n+            }\n+            else if (c == self->quotechar)\n+            {\n+                self->state = FIELD;\n+                break;\n+            }\n+            else\n+            {\n+                // Once data begins, parse it as a normal quoted field\n+                self->state = QUOTED_FIELD;\n+            }\n+\n+        case QUOTED_FIELD:\n+            if (c == self->quotechar) // Parse rest of field normally, e.g. \"ab\"c\n+                self->state = FIELD;\n+            else if (c == '\\n')\n+                self->state = QUOTED_FIELD_NEWLINE;\n+            else if (c == '\\r')\n+            {\n+                HANDLE_CR();\n+            }\n+            else\n+            {\n+                PUSH(c);\n+            }\n+            break;\n+\t\t\n+        case COMMENT:\n+            if (c == '\\n')\n+                self->state = START_LINE;\n+            else if (c == '\\r')\n+            {\n+                HANDLE_CR();\n+            }\n+            break; // keep looping until we find a newline\n+\n+        case CARRIAGE_RETURN:\n+            self->state = old_state;\n+            --self->source_pos; // parse the newline in the old state\n+            if (c != '\\n') // CR line terminator\n+            {\n+                --self->source_pos; // backtrack to the carriage return\n+                parse_newline = 1; // explicitly treat the CR as a newline\n+            }\n+            break;\n+        }\n+\n+        ++self->source_pos;\n+    }\n+    \n+    RETURN(0);\n+}\n+\n+long str_to_long(tokenizer_t *self, char *str)\n+{\n+    errno = 0;\n+    char *tmp;\n+    long ret = strtol(str, &tmp, 0);\n+\n+    if (tmp == str || *tmp != '\\0')\n+ \tself->code = CONVERSION_ERROR;\n+    else if (errno == ERANGE)\n+        self->code = OVERFLOW_ERROR;\n+\n+    return ret;\n+}\n+\n+double str_to_double(tokenizer_t *self, char *str)\n+{\n+    char *tmp;\n+    double val;\n+\n+    if (self->use_fast_converter)\n+    {\n+        val = xstrtod(str, &tmp, '.', 'E', ',', 1);\n+\n+        if (*tmp)\n+            self->code = CONVERSION_ERROR;\n+        else if (errno == ERANGE)\n+            self->code = OVERFLOW_ERROR;\n+    }\n+\n+    else\n+    {\n+        val = strtod(str, &tmp);\n+\n+        if (tmp == str || *tmp != 0)\n+            self->code = CONVERSION_ERROR;\n+        else if (errno == ERANGE)\n+            self->code = OVERFLOW_ERROR;\n+    }\n+\n+    return val;\n+}\n+\n+// ---------------------------------------------------------------------------\n+// Implementation of xstrtod\n+\n+//\n+// strtod.c\n+//\n+// Convert string to double\n+//\n+// Copyright (C) 2002 Michael Ringgaard. All rights reserved.\n+//\n+// Redistribution and use in source and binary forms, with or without\n+// modification, are permitted provided that the following conditions\n+// are met:\n+//\n+// 1. Redistributions of source code must retain the above copyright\n+//    notice, this list of conditions and the following disclaimer.\n+// 2. Redistributions in binary form must reproduce the above copyright\n+//    notice, this list of conditions and the following disclaimer in the\n+//    documentation and/or other materials provided with the distribution.\n+// 3. Neither the name of the project nor the names of its contributors\n+//    may be used to endorse or promote products derived from this software\n+//    without specific prior written permission.\n+//\n+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\" AND\n+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n+// ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE\n+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\n+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS\n+// OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)\n+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT\n+// LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY\n+// OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF\n+// SUCH DAMAGE.\n+//\n+// -----------------------------------------------------------------------\n+// Modifications by Warren Weckesser, March 2011:\n+// * Rename strtod() to xstrtod().\n+// * Added decimal and sci arguments.\n+// * Skip trailing spaces.\n+// * Commented out the other functions.\n+// Modifications by Richard T Guy, August 2013:\n+// * Add tsep argument for thousands separator\n+//\n+\n+double xstrtod(const char *str, char **endptr, char decimal,\n+               char sci, char tsep, int skip_trailing)\n+{\n+    double number;\n+    int exponent;\n+    int negative;\n+    char *p = (char *) str;\n+    double p10;\n+    int n;\n+    int num_digits;\n+    int num_decimals;\n+    int max_digits = 17;\n+\n+    errno = 0;\n+\n+    // Skip leading whitespace\n+    while (isspace(*p)) p++;\n+\n+    // Handle optional sign\n+    negative = 0;\n+    switch (*p)\n+    {\n+    case '-': negative = 1; // Fall through to increment position\n+    case '+': p++;\n+    }\n+\n+    number = 0.;\n+    exponent = 0;\n+    num_digits = 0;\n+    num_decimals = 0;\n+\n+    // Process string of digits\n+    while (isdigit(*p))\n+    {\n+        number = number * 10. + (*p - '0');\n+        p++;\n+        num_digits++;\n+\n+        p += (tsep != '\\0' & *p == tsep);\n+    }\n+\n+    // Process decimal part\n+    if (*p == decimal)\n+    {\n+        p++;\n+\n+        while (num_digits < max_digits && isdigit(*p))\n+        {\n+            number = number * 10. + (*p - '0');\n+            p++;\n+            num_digits++;\n+            num_decimals++;\n+        }\n+        \n+        if (num_digits == max_digits) // consume extra decimal digits\n+            while (isdigit(*p))\n+                ++p;\n+\n+        exponent -= num_decimals;\n+    }\n+\n+    if (num_digits == 0)\n+    {\n+        errno = ERANGE;\n+        return 0.0;\n+    }\n+\n+    // Correct for sign\n+    if (negative) number = -number;\n+\n+    // Process an exponent string\n+    if (toupper(*p) == toupper(sci))\n+    {\n+        // Handle optional sign\n+        negative = 0;\n+        switch (*++p)\n+        {\n+        case '-': negative = 1;   // Fall through to increment pos\n+        case '+': p++;\n+        }\n+\n+        // Process string of digits\n+        n = 0;\n+        while (isdigit(*p))\n+        {\n+            n = n * 10 + (*p - '0');\n+            p++;\n+        }\n+\n+        if (negative)\n+            exponent -= n;\n+        else\n+            exponent += n;\n+    }\n+\n+    if (exponent < DBL_MIN_EXP  || exponent > DBL_MAX_EXP)\n+    {\n+        errno = ERANGE;\n+        return HUGE_VAL;\n+    }\n+\n+    // Scale the result\n+    p10 = 10.;\n+    // Cache some powers of 10 in memory\n+    static double e[] = {1., 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10,\n+                  1e11, 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, 1e20,\n+                  1e21, 1e22, 1e23, 1e24, 1e25, 1e26, 1e27, 1e28, 1e29, 1e30,\n+                  1e31, 1e32, 1e33, 1e34, 1e35, 1e36, 1e37, 1e38, 1e39, 1e40,\n+                  1e41, 1e42, 1e43, 1e44, 1e45, 1e46, 1e47, 1e48, 1e49, 1e50};\n+\n+    if (exponent > 0)\n+    {\n+        while (exponent > 50)\n+        {\n+            number *= e[50];\n+            exponent -= 50;\n+        }\n+    \tnumber *= e[exponent];\n+    }\n+    else\n+    {\n+        while (exponent < -50)\n+        {\n+            number /= e[50];\n+            exponent += 50;\n+        }\n+        number /= e[-exponent];\n+    }\n+\n+    if (number == HUGE_VAL)\n+        errno = ERANGE;\n+\n+    if (skip_trailing) {\n+        // Skip trailing whitespace\n+        while (isspace(*p)) p++;\n+    }\n+\n+    if (endptr) *endptr = p;\n+    return number;\n+}\n+\n+void start_iteration(tokenizer_t *self, int col)\n+{\n+    // Begin looping over the column string with index col\n+    self->iter_col = col;\n+    // Start at the initial pointer position\n+    self->curr_pos = self->output_cols[col];\n+}\n+\n+int finished_iteration(tokenizer_t *self)\n+{\n+    // Iteration is finished if we've exceeded the length of the column string\n+    // or the pointer is at an empty byte\n+    return (self->curr_pos - self->output_cols[self->iter_col] >=\n+            self->output_len[self->iter_col] * sizeof(char)\n+\t    || *self->curr_pos == '\\x00');\n+}\n+\n+char *next_field(tokenizer_t *self, int *size)\n+{\n+    char *tmp = self->curr_pos;\n+\n+    // pass through the entire field until reaching the delimiter\n+    while (*self->curr_pos != '\\x00')\n+\t++self->curr_pos;\n+\n+    ++self->curr_pos; // next field begins after the delimiter\n+\n+    if (*tmp == '\\x01') // empty field; this is a hack\n+    {\n+        if (size)\n+            *size = 0;\n+        return self->buf;\n+    }\n+\n+    else\n+    {\n+        if (size)\n+            *size = self->curr_pos - tmp - 1;\n+        return tmp;\n+    }\n+}\n+\n+// memory mapping won't work on Windows","path":"astropy/io/ascii/src/tokenizer.c","position":796,"original_position":775,"commit_id":"c775ae7dbad8198ff013882be7d5292f621f5534","original_commit_id":"44c9192a075f3ee0ead342282e7b91aa63533503","user":{"login":"amras1","id":6729036,"avatar_url":"https://avatars.githubusercontent.com/u/6729036?v=2","gravatar_id":"6d70a4cf269ccff005eeb67c5e1a69f9","url":"https://api.github.com/users/amras1","html_url":"https://github.com/amras1","followers_url":"https://api.github.com/users/amras1/followers","following_url":"https://api.github.com/users/amras1/following{/other_user}","gists_url":"https://api.github.com/users/amras1/gists{/gist_id}","starred_url":"https://api.github.com/users/amras1/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/amras1/subscriptions","organizations_url":"https://api.github.com/users/amras1/orgs","repos_url":"https://api.github.com/users/amras1/repos","events_url":"https://api.github.com/users/amras1/events{/privacy}","received_events_url":"https://api.github.com/users/amras1/received_events","type":"User","site_admin":false},"body":"Interesting, I found that `PyObject_AsReadBuffer` works correctly while `PyObject_GetBuffer` has the same error (`TypeError: 'mmap.mmap' does not have the buffer interface`). Is it possible that `mmap` only works with the deprecated buffer protocol?","created_at":"2014-08-15T20:09:18Z","updated_at":"2014-08-15T20:09:18Z","html_url":"https://github.com/astropy/astropy/pull/2716#discussion_r16312892","pull_request_url":"https://api.github.com/repos/astropy/astropy/pulls/2716","_links":{"self":{"href":"https://api.github.com/repos/astropy/astropy/pulls/comments/16312892"},"html":{"href":"https://github.com/astropy/astropy/pull/2716#discussion_r16312892"},"pull_request":{"href":"https://api.github.com/repos/astropy/astropy/pulls/2716"}}},"pull_request":{"url":"https://api.github.com/repos/astropy/astropy/pulls/2716","id":18184163,"html_url":"https://github.com/astropy/astropy/pull/2716","diff_url":"https://github.com/astropy/astropy/pull/2716.diff","patch_url":"https://github.com/astropy/astropy/pull/2716.patch","issue_url":"https://api.github.com/repos/astropy/astropy/issues/2716","number":2716,"state":"open","locked":false,"title":"Added a new fast reading/writing engine to io.ascii","user":{"login":"amras1","id":6729036,"avatar_url":"https://avatars.githubusercontent.com/u/6729036?v=2","gravatar_id":"6d70a4cf269ccff005eeb67c5e1a69f9","url":"https://api.github.com/users/amras1","html_url":"https://github.com/amras1","followers_url":"https://api.github.com/users/amras1/followers","following_url":"https://api.github.com/users/amras1/following{/other_user}","gists_url":"https://api.github.com/users/amras1/gists{/gist_id}","starred_url":"https://api.github.com/users/amras1/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/amras1/subscriptions","organizations_url":"https://api.github.com/users/amras1/orgs","repos_url":"https://api.github.com/users/amras1/repos","events_url":"https://api.github.com/users/amras1/events{/privacy}","received_events_url":"https://api.github.com/users/amras1/received_events","type":"User","site_admin":false},"body":"This PR adds a Cython/C parsing engine to the infrastructure of `io.ascii` so that new reading classes for simple formats (currently `FastBasic`, `FastTab`, `FastCsv`, `FastNoHeader`, `FastCommentedHeader`, and `FastRdb`) can opt to use this engine instead of the ordinary `BaseReader` inheritance model. The ordinary readers for these formats still remain to maintain flexibility and compatibility. The PR introduces a new parameter ``use_fast_reader`` in `ascii.read()` which can be used to enable or disable the fast engine (by default, `read()` will attempt to use the fast reader and fall back on the slow reader if parsing fails). The fast readers are also on the guess list directly in front of their slower counterparts.\r\n\r\nThere is also a Cython engine for writing with these formats; a similar parameter ``use_fast_writer`` can enable or disable this engine, and `ascii.write()` uses the engine by default when writing output in a compatible format.\r\n\r\nUsing randomly generated text files, it seems that the fast reading engine is anywhere from 3 to 7 times faster than the ordinary readers depending on the data type of table columns, while the fast writing engine is anywhere from ~2.5 to 12 times faster than the ordinary writers; both engines have less of a speed gain for floating-point data, which might be worth looking into at some point. [Here](http://amras1.github.io/astropy-benchmarks/#) are some before and after benchmarks, which cover a range of ASCII formats and compare Astropy's performance with that of numpy and Pandas. There is also a short design document [here](https://github.com/amras1/astropy-benchmarks/blob/master/design.md), which gives a quick overview of the parsing implementation in this PR.\r\n\r\nPlease feel free to make any suggestions or ask me anything.","created_at":"2014-07-09T20:21:52Z","updated_at":"2014-08-15T20:09:18Z","closed_at":null,"merged_at":null,"merge_commit_sha":"9f2c707ddd57414ad37de73a4806e453de40ec5d","assignee":null,"milestone":{"url":"https://api.github.com/repos/astropy/astropy/milestones/15","labels_url":"https://api.github.com/repos/astropy/astropy/milestones/15/labels","id":643887,"number":15,"title":"v1.0.0","description":"","creator":{"login":"astrofrog","id":314716,"avatar_url":"https://avatars.githubusercontent.com/u/314716?v=2","gravatar_id":"f586e2a9879fe0a55fd1e3ea529c664f","url":"https://api.github.com/users/astrofrog","html_url":"https://github.com/astrofrog","followers_url":"https://api.github.com/users/astrofrog/followers","following_url":"https://api.github.com/users/astrofrog/following{/other_user}","gists_url":"https://api.github.com/users/astrofrog/gists{/gist_id}","starred_url":"https://api.github.com/users/astrofrog/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/astrofrog/subscriptions","organizations_url":"https://api.github.com/users/astrofrog/orgs","repos_url":"https://api.github.com/users/astrofrog/repos","events_url":"https://api.github.com/users/astrofrog/events{/privacy}","received_events_url":"https://api.github.com/users/astrofrog/received_events","type":"User","site_admin":false},"open_issues":130,"closed_issues":14,"state":"open","created_at":"2014-04-29T16:03:02Z","updated_at":"2014-08-15T17:43:26Z","due_on":"2014-11-17T08:00:00Z"},"commits_url":"https://api.github.com/repos/astropy/astropy/pulls/2716/commits","review_comments_url":"https://api.github.com/repos/astropy/astropy/pulls/2716/comments","review_comment_url":"https://api.github.com/repos/astropy/astropy/pulls/comments/{number}","comments_url":"https://api.github.com/repos/astropy/astropy/issues/2716/comments","statuses_url":"https://api.github.com/repos/astropy/astropy/statuses/c775ae7dbad8198ff013882be7d5292f621f5534","head":{"label":"amras1:fast-c-reader","ref":"fast-c-reader","sha":"c775ae7dbad8198ff013882be7d5292f621f5534","user":{"login":"amras1","id":6729036,"avatar_url":"https://avatars.githubusercontent.com/u/6729036?v=2","gravatar_id":"6d70a4cf269ccff005eeb67c5e1a69f9","url":"https://api.github.com/users/amras1","html_url":"https://github.com/amras1","followers_url":"https://api.github.com/users/amras1/followers","following_url":"https://api.github.com/users/amras1/following{/other_user}","gists_url":"https://api.github.com/users/amras1/gists{/gist_id}","starred_url":"https://api.github.com/users/amras1/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/amras1/subscriptions","organizations_url":"https://api.github.com/users/amras1/orgs","repos_url":"https://api.github.com/users/amras1/repos","events_url":"https://api.github.com/users/amras1/events{/privacy}","received_events_url":"https://api.github.com/users/amras1/received_events","type":"User","site_admin":false},"repo":{"id":16990852,"name":"astropy","full_name":"amras1/astropy","owner":{"login":"amras1","id":6729036,"avatar_url":"https://avatars.githubusercontent.com/u/6729036?v=2","gravatar_id":"6d70a4cf269ccff005eeb67c5e1a69f9","url":"https://api.github.com/users/amras1","html_url":"https://github.com/amras1","followers_url":"https://api.github.com/users/amras1/followers","following_url":"https://api.github.com/users/amras1/following{/other_user}","gists_url":"https://api.github.com/users/amras1/gists{/gist_id}","starred_url":"https://api.github.com/users/amras1/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/amras1/subscriptions","organizations_url":"https://api.github.com/users/amras1/orgs","repos_url":"https://api.github.com/users/amras1/repos","events_url":"https://api.github.com/users/amras1/events{/privacy}","received_events_url":"https://api.github.com/users/amras1/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/amras1/astropy","description":"Main Astropy repository","fork":true,"url":"https://api.github.com/repos/amras1/astropy","forks_url":"https://api.github.com/repos/amras1/astropy/forks","keys_url":"https://api.github.com/repos/amras1/astropy/keys{/key_id}","collaborators_url":"https://api.github.com/repos/amras1/astropy/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/amras1/astropy/teams","hooks_url":"https://api.github.com/repos/amras1/astropy/hooks","issue_events_url":"https://api.github.com/repos/amras1/astropy/issues/events{/number}","events_url":"https://api.github.com/repos/amras1/astropy/events","assignees_url":"https://api.github.com/repos/amras1/astropy/assignees{/user}","branches_url":"https://api.github.com/repos/amras1/astropy/branches{/branch}","tags_url":"https://api.github.com/repos/amras1/astropy/tags","blobs_url":"https://api.github.com/repos/amras1/astropy/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/amras1/astropy/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/amras1/astropy/git/refs{/sha}","trees_url":"https://api.github.com/repos/amras1/astropy/git/trees{/sha}","statuses_url":"https://api.github.com/repos/amras1/astropy/statuses/{sha}","languages_url":"https://api.github.com/repos/amras1/astropy/languages","stargazers_url":"https://api.github.com/repos/amras1/astropy/stargazers","contributors_url":"https://api.github.com/repos/amras1/astropy/contributors","subscribers_url":"https://api.github.com/repos/amras1/astropy/subscribers","subscription_url":"https://api.github.com/repos/amras1/astropy/subscription","commits_url":"https://api.github.com/repos/amras1/astropy/commits{/sha}","git_commits_url":"https://api.github.com/repos/amras1/astropy/git/commits{/sha}","comments_url":"https://api.github.com/repos/amras1/astropy/comments{/number}","issue_comment_url":"https://api.github.com/repos/amras1/astropy/issues/comments/{number}","contents_url":"https://api.github.com/repos/amras1/astropy/contents/{+path}","compare_url":"https://api.github.com/repos/amras1/astropy/compare/{base}...{head}","merges_url":"https://api.github.com/repos/amras1/astropy/merges","archive_url":"https://api.github.com/repos/amras1/astropy/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/amras1/astropy/downloads","issues_url":"https://api.github.com/repos/amras1/astropy/issues{/number}","pulls_url":"https://api.github.com/repos/amras1/astropy/pulls{/number}","milestones_url":"https://api.github.com/repos/amras1/astropy/milestones{/number}","notifications_url":"https://api.github.com/repos/amras1/astropy/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/amras1/astropy/labels{/name}","releases_url":"https://api.github.com/repos/amras1/astropy/releases{/id}","created_at":"2014-02-19T16:35:18Z","updated_at":"2014-06-30T18:57:33Z","pushed_at":"2014-08-15T15:39:35Z","git_url":"git://github.com/amras1/astropy.git","ssh_url":"git@github.com:amras1/astropy.git","clone_url":"https://github.com/amras1/astropy.git","svn_url":"https://github.com/amras1/astropy","homepage":"astropy.org","size":29690,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"fast-c-reader"}},"base":{"label":"astropy:master","ref":"master","sha":"c0183e8407785863d2ceeaaf470dcefcb8b945a3","user":{"login":"astropy","id":847984,"avatar_url":"https://avatars.githubusercontent.com/u/847984?v=2","gravatar_id":"df333e02889464493106cdf6de217066","url":"https://api.github.com/users/astropy","html_url":"https://github.com/astropy","followers_url":"https://api.github.com/users/astropy/followers","following_url":"https://api.github.com/users/astropy/following{/other_user}","gists_url":"https://api.github.com/users/astropy/gists{/gist_id}","starred_url":"https://api.github.com/users/astropy/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/astropy/subscriptions","organizations_url":"https://api.github.com/users/astropy/orgs","repos_url":"https://api.github.com/users/astropy/repos","events_url":"https://api.github.com/users/astropy/events{/privacy}","received_events_url":"https://api.github.com/users/astropy/received_events","type":"Organization","site_admin":false},"repo":{"id":2081289,"name":"astropy","full_name":"astropy/astropy","owner":{"login":"astropy","id":847984,"avatar_url":"https://avatars.githubusercontent.com/u/847984?v=2","gravatar_id":"df333e02889464493106cdf6de217066","url":"https://api.github.com/users/astropy","html_url":"https://github.com/astropy","followers_url":"https://api.github.com/users/astropy/followers","following_url":"https://api.github.com/users/astropy/following{/other_user}","gists_url":"https://api.github.com/users/astropy/gists{/gist_id}","starred_url":"https://api.github.com/users/astropy/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/astropy/subscriptions","organizations_url":"https://api.github.com/users/astropy/orgs","repos_url":"https://api.github.com/users/astropy/repos","events_url":"https://api.github.com/users/astropy/events{/privacy}","received_events_url":"https://api.github.com/users/astropy/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/astropy/astropy","description":"Repository for the Astropy core package","fork":false,"url":"https://api.github.com/repos/astropy/astropy","forks_url":"https://api.github.com/repos/astropy/astropy/forks","keys_url":"https://api.github.com/repos/astropy/astropy/keys{/key_id}","collaborators_url":"https://api.github.com/repos/astropy/astropy/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/astropy/astropy/teams","hooks_url":"https://api.github.com/repos/astropy/astropy/hooks","issue_events_url":"https://api.github.com/repos/astropy/astropy/issues/events{/number}","events_url":"https://api.github.com/repos/astropy/astropy/events","assignees_url":"https://api.github.com/repos/astropy/astropy/assignees{/user}","branches_url":"https://api.github.com/repos/astropy/astropy/branches{/branch}","tags_url":"https://api.github.com/repos/astropy/astropy/tags","blobs_url":"https://api.github.com/repos/astropy/astropy/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/astropy/astropy/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/astropy/astropy/git/refs{/sha}","trees_url":"https://api.github.com/repos/astropy/astropy/git/trees{/sha}","statuses_url":"https://api.github.com/repos/astropy/astropy/statuses/{sha}","languages_url":"https://api.github.com/repos/astropy/astropy/languages","stargazers_url":"https://api.github.com/repos/astropy/astropy/stargazers","contributors_url":"https://api.github.com/repos/astropy/astropy/contributors","subscribers_url":"https://api.github.com/repos/astropy/astropy/subscribers","subscription_url":"https://api.github.com/repos/astropy/astropy/subscription","commits_url":"https://api.github.com/repos/astropy/astropy/commits{/sha}","git_commits_url":"https://api.github.com/repos/astropy/astropy/git/commits{/sha}","comments_url":"https://api.github.com/repos/astropy/astropy/comments{/number}","issue_comment_url":"https://api.github.com/repos/astropy/astropy/issues/comments/{number}","contents_url":"https://api.github.com/repos/astropy/astropy/contents/{+path}","compare_url":"https://api.github.com/repos/astropy/astropy/compare/{base}...{head}","merges_url":"https://api.github.com/repos/astropy/astropy/merges","archive_url":"https://api.github.com/repos/astropy/astropy/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/astropy/astropy/downloads","issues_url":"https://api.github.com/repos/astropy/astropy/issues{/number}","pulls_url":"https://api.github.com/repos/astropy/astropy/pulls{/number}","milestones_url":"https://api.github.com/repos/astropy/astropy/milestones{/number}","notifications_url":"https://api.github.com/repos/astropy/astropy/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/astropy/astropy/labels{/name}","releases_url":"https://api.github.com/repos/astropy/astropy/releases{/id}","created_at":"2011-07-21T01:33:49Z","updated_at":"2014-08-15T08:33:41Z","pushed_at":"2014-08-15T17:43:27Z","git_url":"git://github.com/astropy/astropy.git","ssh_url":"git@github.com:astropy/astropy.git","clone_url":"https://github.com/astropy/astropy.git","svn_url":"https://github.com/astropy/astropy","homepage":"www.astropy.org","size":65923,"stargazers_count":577,"watchers_count":577,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"forks_count":284,"mirror_url":null,"open_issues_count":360,"forks":284,"open_issues":360,"watchers":577,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/astropy/astropy/pulls/2716"},"html":{"href":"https://github.com/astropy/astropy/pull/2716"},"issue":{"href":"https://api.github.com/repos/astropy/astropy/issues/2716"},"comments":{"href":"https://api.github.com/repos/astropy/astropy/issues/2716/comments"},"review_comments":{"href":"https://api.github.com/repos/astropy/astropy/pulls/2716/comments"},"review_comment":{"href":"https://api.github.com/repos/astropy/astropy/pulls/comments/{number}"},"commits":{"href":"https://api.github.com/repos/astropy/astropy/pulls/2716/commits"},"statuses":{"href":"https://api.github.com/repos/astropy/astropy/statuses/c775ae7dbad8198ff013882be7d5292f621f5534"}}}}