ronantakizawa/codereview-bench huggingface dataset:
Repository: pandas-dev/pandas
File: pandas/tests/io/parser/test_parse_dates.py (Python)

Code before:
            ["KORD", " 21:56:00", -0.59, 1.71, 5.1, 0.0, 290.0],
        ],
        columns=["X0", "X2", "X3", "X4", "X5", "X6", "X7"],
        index=index,
    )
    if parser.engine == "pyarrow":
        # https://github.com/pandas-dev/pandas/issues/44231
        # pyarrow 6.0 starts to infer time type
        expected["X2"] = pd.to_datetime("1970-01-01" + expected["X2"]).dt.time

    tm.assert_frame_equal(result, expected)


@xfail_pyarrow
def test_nat_parse(all_parsers, temp_file):
    # see gh-3062
    parser = all_parsers
    df = DataFrame(
        {
            "A": np.arange(10, dtype="float64"),
            "B": Timestamp("20010101"),
        }
    )
    df.iloc[3:6, :] = np.nan

    path = temp_file.parent / "__nat_parse_.csv"
    df.to_csv(path)

    result = parser.read_csv(path, index_col=0, parse_dates=["B"])
    tm.assert_frame_equal(result, df)


@skip_pyarrow
def test_parse_dates_implicit_first_col(all_parsers):
    data = """A,B,C
20090101,a,1,2
20090102,b,3,4
20090103,c,4,5
"""
    parser = all_parsers
    result = parser.read_csv(StringIO(data), parse_dates=True)

    expected = parser.read_csv(StringIO(data), index_col=0, parse_dates=True)
    tm.assert_frame_equal(result, expected)


@xfail_pyarrow
def test_parse_dates_string(all_parsers):
    data = """date,A,B,C
20090101,a,1,2
20090102,b,3,4

Review comment (suggestion):
Just use `temp_file` directly (no need to create the `__nat_parse_.csv`

Code after:
            ["KORD", " 21:56:00", -0.59, 1.71, 5.1, 0.0, 290.0],
        ],
        columns=["X0", "X2", "X3", "X4", "X5", "X6", "X7"],
        index=index,
    )
    if parser.engine == "pyarrow":
        # https://github.com/pandas-dev/pandas/issues/44231
        # pyarrow 6.0 starts to infer time type
        expected["X2"] = pd.to_datetime("1970-01-01" + expected["X2"]).dt.time

    tm.assert_frame_equal(result, expected)


@xfail_pyarrow
def test_nat_parse(all_parsers, temp_file):
    # see gh-3062
    parser = all_parsers
    df = DataFrame(
        {
            "A": np.arange(10, dtype="float64"),
            "B": Timestamp("20010101"),
        }
    )
    df.iloc[3:6, :] = np.nan

    path = temp_file
    df.to_csv(path)

    result = parser.read_csv(path, index_col=0, parse_dates=["B"])
    tm.assert_frame_equal(result, df)


@skip_pyarrow
def test_parse_dates_implicit_first_col(all_parsers):
    data = """A,B,C
20090101,a,1,2
20090102,b,3,4
20090103,c,4,5
"""
    parser = all_parsers
    result = parser.read_csv(StringIO(data), parse_dates=True)

    expected = parser.read_csv(StringIO(data), index_col=0, parse_dates=True)
    tm.assert_frame_equal(result, expected)


@xfail_pyarrow
def test_parse_dates_string(all_parsers):
    data = """date,A,B,C
20090101,a,1,2
20090102,b,3,4
Repository: pandas-dev/pandas
File: pandas/tests/tools/test_to_datetime.py (Python)

Code before:
                msg = "Tz-aware datetime.datetime cannot be converted to datetime64"
            with pytest.raises(ValueError, match=msg):
                to_datetime(vec, format="mixed")
            with pytest.raises(ValueError, match=msg):
                DatetimeIndex(vec)
        else:
            if not naive_first and both_datetime:
                msg = "Cannot mix tz-aware with tz-naive values"
            with pytest.raises(ValueError, match=msg):
                to_datetime(vec, format="mixed")
            with pytest.raises(ValueError, match=msg):
                DatetimeIndex(vec)


def test_to_datetime_wrapped_datetime64_ps():
    # GH#60341
    result = to_datetime([np.datetime64(1901901901901, "ps")])
    expected = DatetimeIndex(
        ["1970-01-01 00:00:01.901901901"], dtype="datetime64[ns]", freq=None
    )
    tm.assert_index_equal(result, expected)


def test_to_datetime_lxml_elementunicoderesult_with_format(cache):
    pytest.importorskip("lxml")
    from lxml import etree  # pyright: ignore[reportMissingImports]

    s = "2025-02-05 16:59:57"
    node = etree.XML(f"<date>{s}</date>")
    val = node.xpath("/date/node()")[0]  # _ElementUnicodeResult

    out = to_datetime(Series([val]), format="%Y-%m-%d %H:%M:%S", cache=cache)
    tm.assert_equal(out.iloc[0], Timestamp(s))


Review comment (suggestion):
```suggestion
    etree = pytest.importorskip("lxml.etree")
```

Code after:
                msg = "Tz-aware datetime.datetime cannot be converted to datetime64"
            with pytest.raises(ValueError, match=msg):
                to_datetime(vec, format="mixed")
            with pytest.raises(ValueError, match=msg):
                DatetimeIndex(vec)
        else:
            if not naive_first and both_datetime:
                msg = "Cannot mix tz-aware with tz-naive values"
            with pytest.raises(ValueError, match=msg):
                to_datetime(vec, format="mixed")
            with pytest.raises(ValueError, match=msg):
                DatetimeIndex(vec)


def test_to_datetime_wrapped_datetime64_ps():
    # GH#60341
    result = to_datetime([np.datetime64(1901901901901, "ps")])
    expected = DatetimeIndex(
        ["1970-01-01 00:00:01.901901901"], dtype="datetime64[ns]", freq=None
    )
    tm.assert_index_equal(result, expected)


def test_to_datetime_lxml_elementunicoderesult_with_format(cache):
    etree = pytest.importorskip("lxml.etree")

    s = "2025-02-05 16:59:57"
    node = etree.XML(f"<date>{s}</date>")
    val = node.xpath("/date/node()")[0]  # _ElementUnicodeResult

    out = to_datetime(Series([val]), format="%Y-%m-%d %H:%M:%S", cache=cache)
    assert out.iloc[0] == Timestamp(s)
Repository: pandas-dev/pandas
File: pandas/tests/tools/test_to_datetime.py (Python)

Code before:
                msg = "Cannot mix tz-aware with tz-naive values"
            with pytest.raises(ValueError, match=msg):
                to_datetime(vec, format="mixed")
            with pytest.raises(ValueError, match=msg):
                DatetimeIndex(vec)


def test_to_datetime_wrapped_datetime64_ps():
    # GH#60341
    result = to_datetime([np.datetime64(1901901901901, "ps")])
    expected = DatetimeIndex(
        ["1970-01-01 00:00:01.901901901"], dtype="datetime64[ns]", freq=None
    )
    tm.assert_index_equal(result, expected)


def test_to_datetime_lxml_elementunicoderesult_with_format(cache):
    pytest.importorskip("lxml")
    from lxml import etree  # pyright: ignore[reportMissingImports]

    s = "2025-02-05 16:59:57"
    node = etree.XML(f"<date>{s}</date>")
    val = node.xpath("/date/node()")[0]  # _ElementUnicodeResult

    out = to_datetime(Series([val]), format="%Y-%m-%d %H:%M:%S", cache=cache)
    tm.assert_equal(out.iloc[0], Timestamp(s))


Review comment (suggestion):
```suggestion
    assert out.iloc[0] == Timestamp(s)
```

Code after:
                msg = "Cannot mix tz-aware with tz-naive values"
            with pytest.raises(ValueError, match=msg):
                to_datetime(vec, format="mixed")
            with pytest.raises(ValueError, match=msg):
                DatetimeIndex(vec)


def test_to_datetime_wrapped_datetime64_ps():
    # GH#60341
    result = to_datetime([np.datetime64(1901901901901, "ps")])
    expected = DatetimeIndex(
        ["1970-01-01 00:00:01.901901901"], dtype="datetime64[ns]", freq=None
    )
    tm.assert_index_equal(result, expected)


def test_to_datetime_lxml_elementunicoderesult_with_format(cache):
    etree = pytest.importorskip("lxml.etree")

    s = "2025-02-05 16:59:57"
    node = etree.XML(f"<date>{s}</date>")
    val = node.xpath("/date/node()")[0]  # _ElementUnicodeResult

    out = to_datetime(Series([val]), format="%Y-%m-%d %H:%M:%S", cache=cache)
    assert out.iloc[0] == Timestamp(s)
Repository: pandas-dev/pandas
File: pandas/_libs/include/pandas/parser/tokenizer.h (C/C++)

Code before:

void parser_del(parser_t *self);

void parser_set_default_options(parser_t *self);

int tokenize_nrows(parser_t *self, size_t nrows, const char *encoding_errors);

int tokenize_all_rows(parser_t *self, const char *encoding_errors);

// Have parsed / type-converted a chunk of data
// and want to free memory from the token stream

typedef struct uint_state {
  int seen_sint;
  int seen_uint;
  int seen_null;
} uint_state;

void uint_state_init(uint_state *self);

int uint64_conflict(uint_state *self);

uint64_t str_to_uint64(uint_state *state, const char *p_item,
                       char decimal_separator, int64_t int_max,
                       uint64_t uint_max, int *error, char tsep);
int64_t str_to_int64(const char *p_item, char decimal_separator,
                     int64_t int_min, int64_t int_max, int *error, char tsep);
double xstrtod(const char *p, char **q, char decimal, char sci, char tsep,
               int skip_trailing, int *error, int *maybe_int);
double precise_xstrtod(const char *p, char **q, char decimal, char sci,
                       char tsep, int skip_trailing, int *error,
                       int *maybe_int);

// GH-15140 - round_trip requires and acquires the GIL on its own
double round_trip(const char *p, char **q, char decimal, char sci, char tsep,
                  int skip_trailing, int *error, int *maybe_int);
int to_boolean(const char *item, uint8_t *val);


Review comment (suggestion):
I don't think we should add this argument to the int conversion functions - that repurposes these functions in a way that's not really clear.

Code after:
void parser_del(parser_t *self);

void parser_set_default_options(parser_t *self);

int tokenize_nrows(parser_t *self, size_t nrows, const char *encoding_errors);

int tokenize_all_rows(parser_t *self, const char *encoding_errors);

// Have parsed / type-converted a chunk of data
// and want to free memory from the token stream

typedef struct uint_state {
  int seen_sint;
  int seen_uint;
  int seen_null;
} uint_state;

void uint_state_init(uint_state *self);

int uint64_conflict(uint_state *self);

uint64_t str_to_uint64(uint_state *state, const char *p_item, int *error,
                       char tsep);
int64_t str_to_int64(const char *p_item, int *error, char tsep);
double xstrtod(const char *p, char **q, char decimal, char sci, char tsep,
               int skip_trailing, int *error, int *maybe_int);
double precise_xstrtod(const char *p, char **q, char decimal, char sci,
                       char tsep, int skip_trailing, int *error,
                       int *maybe_int);

// GH-15140 - round_trip requires and acquires the GIL on its own
double round_trip(const char *p, char **q, char decimal, char sci, char tsep,
                  int skip_trailing, int *error, int *maybe_int);
int to_boolean(const char *item, uint8_t *val);
Repository: pandas-dev/pandas
File: pandas/_libs/src/parser/tokenizer.c (C)

Code before:
  }

  int64_t number = 0;
  if (isneg) {
    // If number is greater than pre_min, at least one more digit
    // can be processed without overflowing.
    int dig_pre_min = -(int_min % 10);
    int64_t pre_min = int_min / 10;

    // Process the digits.
    char d = *p;
    if (tsep != '\0') {
      while (1) {
        if (d == tsep) {
          d = *++p;
          continue;
        } else if (!isdigit_ascii(d)) {
          break;
        }
        if ((number > pre_min) ||
            ((number == pre_min) && (d - '0' <= dig_pre_min))) {
          number = number * 10 - (d - '0');
          d = *++p;
        } else {
          *error = ERROR_OVERFLOW;
          break;
        }
      }
    } else {
      while (isdigit_ascii(d)) {
        if ((number > pre_min) ||
            ((number == pre_min) && (d - '0' <= dig_pre_min))) {
          number = number * 10 - (d - '0');
          d = *++p;
        } else {
          *error = ERROR_OVERFLOW;
          break;
        }
      }
    }
  } else {
    // If number is less than pre_max, at least one more digit
    // can be processed without overflowing.
    int64_t pre_max = int_max / 10;
    int dig_pre_max = int_max % 10;

    // Process the digits.
    char d = *p;
    if (tsep != '\0') {
      while (1) {
        if (d == tsep) {

Review comment (bug):
Can we try to keep these are immediate returns? This opens up the door to ambiguous behavior of the parser in case of "multiple" failures

Code after:
      right = end_ptr;
    }

    const size_t chunk_size = right - left;

    if (chunk_size + bytes_written >= PROCESSED_WORD_CAPACITY) {
      return -1;
    }
    memcpy(&output[bytes_written], left, chunk_size);
    bytes_written += chunk_size;

    left = right + 1;
  }

  output[bytes_written] = '\0';
  return 0;
}

int64_t str_to_int64(const char *p_item, int *error, char tsep) {
  const char *p = p_item;
  // Skip leading spaces.
  while (isspace_ascii(*p)) {
    ++p;
  }

  // Handle sign.
  const bool has_sign = *p == '-' || *p == '+';
  // Handle sign.
  const char *digit_start = has_sign ? p + 1 : p;

  // Check that there is a first digit.
  if (!isdigit_ascii(*digit_start)) {
    // Error...
    *error = ERROR_NO_DIGITS;
    return 0;
  }

  char buffer[PROCESSED_WORD_CAPACITY];
  const size_t str_len = strlen(p);
  if (tsep != '\0' && memchr(p, tsep, str_len) != NULL) {
    const int status = copy_string_without_char(buffer, p, str_len, tsep);
    if (status != 0) {
      // Word is too big, probably will cause an overflow
      *error = ERROR_OVERFLOW;
      return 0;
    }
    p = buffer;
  }

  char *endptr;
  int64_t number = strtoll(p, &endptr, 10);
Repository: pandas-dev/pandas
File: pandas/_libs/src/parser/tokenizer.c (C)

Code before:
void uint_state_init(uint_state *self) {
  self->seen_sint = 0;
  self->seen_uint = 0;
  self->seen_null = 0;
}

int uint64_conflict(uint_state *self) {
  return self->seen_uint && (self->seen_sint || self->seen_null);
}

/**
 * @brief Validates that a string contains only numeric digits.
 *
 * This function is used after an integer overflow,
 * where is checks the rest of the string for a non-numeric character.
 *
 * Pure integer overflows during CSV parsing are converted to PyLongObjects,
 * while, if any invalid character is found, it skips integer
 * parsing and tries other conversion methods.
 *
 * @param p_item Pointer to the string to validate for numeric format
 *
 * @return Integer 0 if the remainder of the string contains only digits,
 *         otherwise returns the error code for [ERROR_INVALID_CHARS].
 */
static inline int check_for_invalid_char(const char *p_item) {
  while (*p_item != '\0' && isdigit_ascii(*p_item)) {
    p_item++;
  }

  // check if reached the end of string after consuming all digits
  if (*p_item != '\0') {
    return ERROR_INVALID_CHARS;
  }

  return 0;
}

int64_t str_to_int64(const char *p_item, int64_t int_min, int64_t int_max,
                     int *error, char tsep) {
  const char *p = p_item;
  // Skip leading spaces.
  while (isspace_ascii(*p)) {
    ++p;
  }

  // Handle sign.
  const bool isneg = *p == '-' ? true : false;
  // Handle sign.
  if (isneg || (*p == '+')) {
    p++;

Review comment (refactor):
Can you add the length of the string as an argument? I realize this is a static function, but its still best to guard against buffer overruns in case of future refactor

Code after:
}

// End of xstrtod code
// ---------------------------------------------------------------------------

void uint_state_init(uint_state *self) {
  self->seen_sint = 0;
  self->seen_uint = 0;
  self->seen_null = 0;
}

int uint64_conflict(uint_state *self) {
  return self->seen_uint && (self->seen_sint || self->seen_null);
}

/* Copy a string without `char_to_remove` into `output`.
 */
static int copy_string_without_char(char output[PROCESSED_WORD_CAPACITY],
                                    const char *str, size_t str_len,
                                    char char_to_remove) {
  const char *left = str;
  const char *end_ptr = str + str_len;
  size_t bytes_written = 0;

  while (left < end_ptr) {
    const size_t remaining_bytes_to_read = end_ptr - left;
    const char *right = memchr(left, char_to_remove, remaining_bytes_to_read);

    if (!right) {
      // If it doesn't find the char to remove, just copy until EOS.
      right = end_ptr;
    }

    const size_t chunk_size = right - left;

    if (chunk_size + bytes_written >= PROCESSED_WORD_CAPACITY) {
      return -1;
    }
    memcpy(&output[bytes_written], left, chunk_size);
    bytes_written += chunk_size;

    left = right + 1;
  }

  output[bytes_written] = '\0';
  return 0;
}

int64_t str_to_int64(const char *p_item, int *error, char tsep) {
  const char *p = p_item;
  // Skip leading spaces.
Repository: pandas-dev/pandas
File: pandas/_libs/src/parser/tokenizer.c (C)

Code before:
void uint_state_init(uint_state *self) {
  self->seen_sint = 0;
  self->seen_uint = 0;
  self->seen_null = 0;
}

int uint64_conflict(uint_state *self) {
  return self->seen_uint && (self->seen_sint || self->seen_null);
}

/**
 * @brief Validates that a string contains only numeric digits.
 *
 * This function is used after an integer overflow,
 * where is checks the rest of the string for a non-numeric character.
 *
 * Pure integer overflows during CSV parsing are converted to PyLongObjects,
 * while, if any invalid character is found, it skips integer
 * parsing and tries other conversion methods.
 *
 * @param p_item Pointer to the string to validate for numeric format
 *
 * @return TOKENIZER_OK if the remainder of the string contains only digits,
 *         otherwise returns ERROR_INVALID_CHARS.
 */
static inline TokenizerError check_for_invalid_char(const char *p_item) {
  while (*p_item != '\0' && isdigit_ascii(*p_item)) {
    p_item++;
  }

  // check if reached the end of string after consuming all digits
  if (*p_item != '\0') {
    return ERROR_INVALID_CHARS;
  }

  return TOKENIZER_OK;
}

int64_t str_to_int64(const char *p_item, int64_t int_min, int64_t int_max,
                     TokenizerError *error, char tsep) {
  const char *p = p_item;
  // Skip leading spaces.
  while (isspace_ascii(*p)) {
    ++p;
  }

  // Handle sign.
  const bool isneg = *p == '-' ? true : false;
  // Handle sign.
  if (isneg || (*p == '+')) {
    p++;

Review comment (suggestion):
I think my comment got lost but you should add the length of the string to the arguments here, so that we can mitigate the risk of buffer overruns. 

Code after:
}

// End of xstrtod code
// ---------------------------------------------------------------------------

void uint_state_init(uint_state *self) {
  self->seen_sint = 0;
  self->seen_uint = 0;
  self->seen_null = 0;
}

int uint64_conflict(uint_state *self) {
  return self->seen_uint && (self->seen_sint || self->seen_null);
}

/* Copy a string without `char_to_remove` into `output`.
 */
static int copy_string_without_char(char output[PROCESSED_WORD_CAPACITY],
                                    const char *str, size_t str_len,
                                    char char_to_remove) {
  const char *left = str;
  const char *end_ptr = str + str_len;
  size_t bytes_written = 0;

  while (left < end_ptr) {
    const size_t remaining_bytes_to_read = end_ptr - left;
    const char *right = memchr(left, char_to_remove, remaining_bytes_to_read);

    if (!right) {
      // If it doesn't find the char to remove, just copy until EOS.
      right = end_ptr;
    }

    const size_t chunk_size = right - left;

    if (chunk_size + bytes_written >= PROCESSED_WORD_CAPACITY) {
      return -1;
    }
    memcpy(&output[bytes_written], left, chunk_size);
    bytes_written += chunk_size;

    left = right + 1;
  }

  output[bytes_written] = '\0';
  return 0;
}

int64_t str_to_int64(const char *p_item, int *error, char tsep) {
  const char *p = p_item;
  // Skip leading spaces.
Repository: pandas-dev/pandas
File: pandas/_libs/src/parser/tokenizer.c (C)

Code before:
  }

  int64_t number = 0;
  if (isneg) {
    // If number is greater than pre_min, at least one more digit
    // can be processed without overflowing.
    int dig_pre_min = -(int_min % 10);
    int64_t pre_min = int_min / 10;

    // Process the digits.
    char d = *p;
    if (tsep != '\0') {
      while (1) {
        if (d == tsep) {
          d = *++p;
          continue;
        } else if (!isdigit_ascii(d)) {
          break;
        }
        if ((number > pre_min) ||
            ((number == pre_min) && (d - '0' <= dig_pre_min))) {
          number = number * 10 - (d - '0');
          d = *++p;
        } else {
          TokenizerError status;
          if ((status = check_for_invalid_char(p)) > ERROR_OVERFLOW) {
            *error = status;
          } else {
            *error = ERROR_OVERFLOW;
          }
          return 0;
        }
      }
    } else {
      while (isdigit_ascii(d)) {
        if ((number > pre_min) ||
            ((number == pre_min) && (d - '0' <= dig_pre_min))) {
          number = number * 10 - (d - '0');
          d = *++p;
        } else {
          TokenizerError status;
          if ((status = check_for_invalid_char(p)) > ERROR_OVERFLOW) {
            *error = status;
          } else {
            *error = ERROR_OVERFLOW;
          }
          return 0;
        }
      }
    }
  } else {

Review comment (suggestion):
Adding ordering semantics to the enum isn't really clear on the intent. Probably best to make the function check for more than invalid chars and return the appropriate error code, rather than handling this logic multiple times in the caller

Code after:
  const char *digit_start = has_sign ? p + 1 : p;

  // Check that there is a first digit.
  if (!isdigit_ascii(*digit_start)) {
    // Error...
    *error = ERROR_NO_DIGITS;
    return 0;
  }

  char buffer[PROCESSED_WORD_CAPACITY];
  const size_t str_len = strlen(p);
  if (tsep != '\0' && memchr(p, tsep, str_len) != NULL) {
    const int status = copy_string_without_char(buffer, p, str_len, tsep);
    if (status != 0) {
      // Word is too big, probably will cause an overflow
      *error = ERROR_OVERFLOW;
      return 0;
    }
    p = buffer;
  }

  char *endptr;
  int64_t number = strtoll(p, &endptr, 10);

  if (errno == ERANGE) {
    *error = *endptr ? ERROR_INVALID_CHARS : ERROR_OVERFLOW;
    errno = 0;
    return 0;
  }

  // Skip trailing spaces.
  while (isspace_ascii(*endptr)) {
    ++endptr;
  }

  // Did we use up all the characters?
  if (*endptr) {
    *error = ERROR_INVALID_CHARS;
    return 0;
  }

  *error = 0;
  return number;
}

uint64_t str_to_uint64(uint_state *state, const char *p_item, int *error,
                       char tsep) {
  const char *p = p_item;
  // Skip leading spaces.
  while (isspace_ascii(*p)) {
    ++p;