Repository: pandas-dev/pandas
File: pandas/_libs/src/parser/tokenizer.c (C)

Code before:
  }

  int64_t number = 0;
  if (isneg) {
    // If number is greater than pre_min, at least one more digit
    // can be processed without overflowing.
    int dig_pre_min = -(int_min % 10);
    int64_t pre_min = int_min / 10;

    // Process the digits.
    char d = *p;
    if (tsep != '\0') {
      while (1) {
        if (d == tsep) {
          d = *++p;
          continue;
        } else if (!isdigit_ascii(d)) {
          break;
        }
        if ((number > pre_min) ||
            ((number == pre_min) && (d - '0' <= dig_pre_min))) {
          number = number * 10 - (d - '0');
          d = *++p;
        } else {
          *error = ERROR_OVERFLOW;
          int status = check_for_invalid_char(p);
          if (status != 0) {
            *error = status;
          }
          return 0;
        }
      }
    } else {
      while (isdigit_ascii(d)) {
        if ((number > pre_min) ||
            ((number == pre_min) && (d - '0' <= dig_pre_min))) {
          number = number * 10 - (d - '0');
          d = *++p;
        } else {
          *error = ERROR_OVERFLOW;
          int status = check_for_invalid_char(p);
          if (status != 0) {
            *error = status;
          }
          return 0;
        }
      }
    }
  } else {
    // If number is less than pre_max, at least one more digit
    // can be processed without overflowing.

Review comment (bug):
I'm not sure why tokenizer.h defines errors individually, but it would be much easier if you created a struct like:

```c
enum TokenizerError {
    TOKENIZER_OK,
    ERROR_NO_DIGITS,
    ERROR_OVERFLOW,
    ERROR_INVALID_CHARS
};
```

Then you can just assign `*error = check_for_invalid_char(p)` and let the call stack naturally handle this (where 0 is no error).

Although its still a bit strange to apply an error on the preceding line then reassign it here. I wonder if there shouldn't be a generic function to check for errors here

Code after:
  const char *digit_start = has_sign ? p + 1 : p;

  // Check that there is a first digit.
  if (!isdigit_ascii(*digit_start)) {
    // Error...
    *error = ERROR_NO_DIGITS;
    return 0;
  }

  char buffer[PROCESSED_WORD_CAPACITY];
  const size_t str_len = strlen(p);
  if (tsep != '\0' && memchr(p, tsep, str_len) != NULL) {
    const int status = copy_string_without_char(buffer, p, str_len, tsep);
    if (status != 0) {
      // Word is too big, probably will cause an overflow
      *error = ERROR_OVERFLOW;
      return 0;
    }
    p = buffer;
  }

  char *endptr;
  int64_t number = strtoll(p, &endptr, 10);

  if (errno == ERANGE) {
    *error = *endptr ? ERROR_INVALID_CHARS : ERROR_OVERFLOW;
    errno = 0;
    return 0;
  }

  // Skip trailing spaces.
  while (isspace_ascii(*endptr)) {
    ++endptr;
  }

  // Did we use up all the characters?
  if (*endptr) {
    *error = ERROR_INVALID_CHARS;
    return 0;
  }

  *error = 0;
  return number;
}

uint64_t str_to_uint64(uint_state *state, const char *p_item, int *error,
                       char tsep) {
  const char *p = p_item;
  // Skip leading spaces.
  while (isspace_ascii(*p)) {
    ++p;