ronantakizawa/codereview-bench huggingface dataset:
Repository: pandas-dev/pandas
File: pandas/_libs/src/parser/tokenizer.c (C)

Code before:
  case '-':
    return str[1] != '\0' && isdigit_ascii(str[1]);
  default:
    return false;
  }
}

static inline bool has_only_spaces(const char *str) {
  while (*str != '\0' && isspace_ascii(*str)) {
    str++;
  }
  return *str == '\0';
}

/* Copy a string without `char_to_remove` into `output`.
 */
static int copy_string_without_char(char output[PROCESSED_WORD_CAPACITY],
                                    const char *str, size_t str_len,
                                    char char_to_remove) {
  const char *left = str;
  const char *right;
  const char *end_ptr = str + str_len;
  size_t bytes_written = 0;

  while (left < end_ptr) {
    size_t remaining_bytes_to_read = end_ptr - left;
    right = memchr(left, char_to_remove, remaining_bytes_to_read);

    if (!right) {
      // If it doesn't find the char to remove, just copy until EOS.
      right = end_ptr;
    }

    size_t chunk_size = right - left;

    // check if we have enough space, including the null terminator.
    if (chunk_size + bytes_written >= PROCESSED_WORD_CAPACITY) {
      return -1;
    }
    // copy block
    memcpy(&output[bytes_written], left, chunk_size);
    bytes_written += chunk_size;

    // Advance past the removed character if we found it.
    left = right + 1;
  }

  // null terminate
  output[bytes_written] = '\0';
  return 0;
}

Review comment (suggestion):
```suggestion
    const size_t remaining_bytes_to_read = end_ptr - left;
```

Always best to add `const` unless there's a reason for mutability

Code after:

    left = right + 1;
  }

  output[bytes_written] = '\0';
  return 0;
}

int64_t str_to_int64(const char *p_item, int64_t int_min, int64_t int_max,
                     int *error, char tsep) {
  const char *p = p_item;
  // Skip leading spaces.
  while (isspace_ascii(*p)) {
    ++p;
  }

  // Handle sign.
  const bool has_sign = *p == '-' || *p == '+';
  // Handle sign.
  const char *digit_start = has_sign ? p + 1 : p;

  // Check that there is a first digit.
  if (!isdigit_ascii(*digit_start)) {
    // Error...
    *error = ERROR_NO_DIGITS;
    return 0;
  }

  char buffer[PROCESSED_WORD_CAPACITY];
  const size_t str_len = strlen(p);
  if (tsep != '\0' && memchr(p, tsep, str_len) != NULL) {
    const int status = copy_string_without_char(buffer, p, str_len, tsep);
    if (status != 0) {
      // Word is too big, probably will cause an overflow
      *error = ERROR_OVERFLOW;
      return 0;
    }
    p = buffer;
  }

  char *endptr;
  int64_t number = strtoll(p, &endptr, 10);

  if (errno == ERANGE || number > int_max || number < int_min) {
    *error = ERROR_OVERFLOW;
    errno = 0;
    return 0;
  }

  // Skip trailing spaces.
  while (isspace_ascii(*endptr)) {
Repository: pandas-dev/pandas
File: pandas/_libs/src/parser/tokenizer.c (C)

Code before:
  while (*str != '\0' && isspace_ascii(*str)) {
    str++;
  }
  return *str == '\0';
}

/* Copy a string without `char_to_remove` into `output`.
 */
static int copy_string_without_char(char output[PROCESSED_WORD_CAPACITY],
                                    const char *str, size_t str_len,
                                    char char_to_remove) {
  const char *left = str;
  const char *right;
  const char *end_ptr = str + str_len;
  size_t bytes_written = 0;

  while (left < end_ptr) {
    size_t remaining_bytes_to_read = end_ptr - left;
    right = memchr(left, char_to_remove, remaining_bytes_to_read);

    if (!right) {
      // If it doesn't find the char to remove, just copy until EOS.
      right = end_ptr;
    }

    size_t chunk_size = right - left;

    // check if we have enough space, including the null terminator.
    if (chunk_size + bytes_written >= PROCESSED_WORD_CAPACITY) {
      return -1;
    }
    // copy block
    memcpy(&output[bytes_written], left, chunk_size);
    bytes_written += chunk_size;

    // Advance past the removed character if we found it.
    left = right + 1;
  }

  // null terminate
  output[bytes_written] = '\0';
  return 0;
}

int64_t str_to_int64(const char *p_item, int64_t int_min, int64_t int_max,
                     int *error, char tsep) {
  if (!p_item || *p_item == '\0') {
    *error = ERROR_NO_DIGITS;
    return 0;
  }


Review comment (suggestion):
```suggestion
    const size_t chunk_size = right - left;
```

Code after:
int64_t str_to_int64(const char *p_item, int64_t int_min, int64_t int_max,
                     int *error, char tsep) {
  const char *p = p_item;
  // Skip leading spaces.
  while (isspace_ascii(*p)) {
    ++p;
  }

  // Handle sign.
  const bool has_sign = *p == '-' || *p == '+';
  // Handle sign.
  const char *digit_start = has_sign ? p + 1 : p;

  // Check that there is a first digit.
  if (!isdigit_ascii(*digit_start)) {
    // Error...
    *error = ERROR_NO_DIGITS;
    return 0;
  }

  char buffer[PROCESSED_WORD_CAPACITY];
  const size_t str_len = strlen(p);
  if (tsep != '\0' && memchr(p, tsep, str_len) != NULL) {
    const int status = copy_string_without_char(buffer, p, str_len, tsep);
    if (status != 0) {
      // Word is too big, probably will cause an overflow
      *error = ERROR_OVERFLOW;
      return 0;
    }
    p = buffer;
  }

  char *endptr;
  int64_t number = strtoll(p, &endptr, 10);

  if (errno == ERANGE || number > int_max || number < int_min) {
    *error = ERROR_OVERFLOW;
    errno = 0;
    return 0;
  }

  // Skip trailing spaces.
  while (isspace_ascii(*endptr)) {
    ++endptr;
  }

  // Did we use up all the characters?
  if (*endptr) {
    *error = ERROR_INVALID_CHARS;
    return 0;
  }
Repository: pandas-dev/pandas
File: pandas/_libs/src/parser/tokenizer.c (C)

Code before:
      // If it doesn't find the char to remove, just copy until EOS.
      right = end_ptr;
    }

    size_t chunk_size = right - left;

    // check if we have enough space, including the null terminator.
    if (chunk_size + bytes_written >= PROCESSED_WORD_CAPACITY) {
      return -1;
    }
    // copy block
    memcpy(&output[bytes_written], left, chunk_size);
    bytes_written += chunk_size;

    // Advance past the removed character if we found it.
    left = right + 1;
  }

  // null terminate
  output[bytes_written] = '\0';
  return 0;
}

int64_t str_to_int64(const char *p_item, int64_t int_min, int64_t int_max,
                     int *error, char tsep) {
  if (!p_item || *p_item == '\0') {
    *error = ERROR_NO_DIGITS;
    return 0;
  }

  while (isspace_ascii(*p_item)) {
    ++p_item;
  }

  if (!has_digit_int(p_item)) {
    *error = ERROR_NO_DIGITS;
    return 0;
  }

  char buffer[PROCESSED_WORD_CAPACITY];
  size_t str_len = strlen(p_item);
  if (tsep != '\0' && memchr(p_item, tsep, str_len) != NULL) {
    int status = copy_string_without_char(buffer, p_item, str_len, tsep);

    if (status != 0) {
      // Word is too big, probably will cause an overflow
      *error = ERROR_OVERFLOW;
      return 0;
    }

    p_item = buffer;

Review comment (suggestion):
Is it necessary to add this? I don't think we should be special-casing behavior for null or the null byte; I expect the former is undefined (since its not a string) and the latter would be handled naturally by the rest of the logic (?)

Code after:
  const size_t str_len = strlen(p);
  if (tsep != '\0' && memchr(p, tsep, str_len) != NULL) {
    const int status = copy_string_without_char(buffer, p, str_len, tsep);
    if (status != 0) {
      // Word is too big, probably will cause an overflow
      *error = ERROR_OVERFLOW;
      return 0;
    }
    p = buffer;
  }

  char *endptr;
  int64_t number = strtoll(p, &endptr, 10);

  if (errno == ERANGE || number > int_max || number < int_min) {
    *error = ERROR_OVERFLOW;
    errno = 0;
    return 0;
  }

  // Skip trailing spaces.
  while (isspace_ascii(*endptr)) {
    ++endptr;
  }

  // Did we use up all the characters?
  if (*endptr) {
    *error = ERROR_INVALID_CHARS;
    return 0;
  }

  *error = 0;
  return number;
}

uint64_t str_to_uint64(uint_state *state, const char *p_item, int64_t int_max,
                       uint64_t uint_max, int *error, char tsep) {
  const char *p = p_item;
  // Skip leading spaces.
  while (isspace_ascii(*p)) {
    ++p;
  }

  // Handle sign.
  if (*p == '-') {
    state->seen_sint = 1;
    *error = 0;
    return 0;
  } else if (*p == '+') {
    p++;
  }
Repository: pandas-dev/pandas
File: pandas/tests/series/methods/test_info.py (Python)

Code before:
    # memory usage introspection should not materialize .values
    N = 100
    M = len(ascii_uppercase)
    index = MultiIndex.from_product(
        [list(ascii_uppercase), date_range("20160101", periods=N)],
        names=["id", "date"],
    )
    s = Series(np.random.default_rng(2).standard_normal(N * M), index=index)

    unstacked = s.unstack("id")
    assert s.values.nbytes == unstacked.values.nbytes
    assert s.memory_usage(deep=True) > unstacked.memory_usage(deep=True).sum()

    # high upper bound
    diff = unstacked.memory_usage(deep=True).sum() - s.memory_usage(deep=True)
    assert diff < 2000

def test_info_show_counts_false():
    # GH#62590
    s = Series([1, 2, None])
    buf = StringIO()
    s.info(buf=buf, show_counts=False)
    result = buf.getvalue()
    assert "Non-Null Count" not in result
    assert "<class 'pandas.Series'>" in result
    assert "Dtype" in result


Review comment (suggestion):
can you follow the pattern in `test_info_memory()` in this file where you just check that the string is what we expect.

ping when green on CI

Code after:
    # memory usage introspection should not materialize .values
    N = 100
    M = len(ascii_uppercase)
    index = MultiIndex.from_product(
        [list(ascii_uppercase), date_range("20160101", periods=N)],
        names=["id", "date"],
    )
    s = Series(np.random.default_rng(2).standard_normal(N * M), index=index)

    unstacked = s.unstack("id")
    assert s.values.nbytes == unstacked.values.nbytes
    assert s.memory_usage(deep=True) > unstacked.memory_usage(deep=True).sum()

    # high upper bound
    diff = unstacked.memory_usage(deep=True).sum() - s.memory_usage(deep=True)
    assert diff < 2000


def test_info_show_counts_false():
    s = Series([1])
    buf = StringIO()
    s.info(buf=buf, show_counts=False)
    result = buf.getvalue()
    memory_bytes = float(s.memory_usage())
    expected = textwrap.dedent(
        f"""\
    <class 'pandas.Series'>
    RangeIndex: 1 entries, 0 to 0
    Series name: None
    Dtype
    -----
    int64
    dtypes: int64(1)
    memory usage: {memory_bytes} bytes"""
    )
    assert result.strip() == expected.strip()
Repository: pandas-dev/pandas
File: pandas/core/groupby/generic.py (Python)

Code before:
        result_a  result_1
    key
    1          -1      10.5
    2           1      12.0

    >>> def n_between(ser, low, high, **kwargs):
    ...     return ser.between(low, high, **kwargs).sum()

    >>> agg_between = pd.NamedAgg("a", n_between, 0, 1)
    >>> df.groupby("key").agg(count_between=agg_between)
        count_between
    key
    1               1
    2               1

    >>> agg_between_kw = pd.NamedAgg("a", n_between, 0, 1, inclusive="both")
    >>> df.groupby("key").agg(count_between_kw=agg_between_kw)
        count_between_kw
    key
    1                   1
    2                   1
    """

    column: Hashable
    aggfunc: AggScalar
    args: tuple[Any, ...] = dataclasses.field(default_factory=tuple)
    kwargs: dict[str, Any] = dataclasses.field(default_factory=dict)

    def __init__(
        self,
        column: Hashable,
        aggfunc: Callable[..., Any] | str,
        *args: Any,
        **kwargs: Any,
    ) -> None:
        self.column = column
        self.aggfunc = aggfunc
        self.args = args
        self.kwargs = kwargs

    def __getitem__(self, key: int) -> Any:
        """Provide backward-compatible tuple-style access."""
        if key == 0:
            return self.column
        elif key == 1:
            return self.aggfunc
        elif key == 2:
            return self.args
        elif key == 3:
            return self.kwargs
        raise IndexError("index out of range")

Review comment (suggestion):
I think either way is fine, just wanted to mention this could be `args: tuple[Any, ...] = ()` since tuples are immutable.

Code after:
        result_a  result_1
    key
    1          -1      10.5
    2           1      12.0

    >>> def n_between(ser, low, high, **kwargs):
    ...     return ser.between(low, high, **kwargs).sum()

    >>> agg_between = pd.NamedAgg("a", n_between, 0, 1)
    >>> df.groupby("key").agg(count_between=agg_between)
        count_between
    key
    1               1
    2               1

    >>> agg_between_kw = pd.NamedAgg("a", n_between, 0, 1, inclusive="both")
    >>> df.groupby("key").agg(count_between_kw=agg_between_kw)
        count_between_kw
    key
    1                   1
    2                   1
    """

    column: Hashable
    aggfunc: AggScalar
    args: tuple[Any, ...] = ()
    kwargs: dict[str, Any] = dataclasses.field(default_factory=dict)

    def __init__(
        self,
        column: Hashable,
        aggfunc: Callable[..., Any] | str,
        *args: Any,
        **kwargs: Any,
    ) -> None:
        self.column = column
        self.aggfunc = aggfunc
        self.args = args
        self.kwargs = kwargs

    def __getitem__(self, key: int) -> Any:
        """Provide backward-compatible tuple-style access."""
        if key == 0:
            return self.column
        elif key == 1:
            return self.aggfunc
        elif key == 2:
            return self.args
        elif key == 3:
            return self.kwargs
        raise IndexError("index out of range")
Repository: pandas-dev/pandas
File: pandas/_libs/src/parser/tokenizer.c (C)

Code before:

    if (num_digits >= max_digits) // Consume extra decimal digits.
      while (isdigit_ascii(*p))
        ++p;

    exponent -= num_decimals;
  }

  if (num_digits == 0) {
    *error = ERANGE;
    return 0.0;
  }

  // Correct for sign.
  if (negative)
    number = -number;

  // Process an exponent string.
  if (toupper_ascii(*p) == toupper_ascii(sci)) {
    if (maybe_int != NULL)
      *maybe_int = 0;

    // Handle optional sign
    negative = 0;
    switch (*++p) {
    case '-':
      negative = 1;
      PD_FALLTHROUGH; // Fall through to increment position.
    case '+':
      p++;
      break;
    }

    // Process string of digits.
    num_digits = 0;
    int n = 0;
    while (num_digits < max_digits && isdigit_ascii(*p)) {
      n = n * 10 + (*p - '0');
      num_digits++;
      p++;
    }

    if (negative)
      exponent -= n;
    else
      exponent += n;

    // If no digits after the 'e'/'E', un-consume it.
    if (num_digits == 0)
      p--;
  }

Review comment (suggestion):
I think this is an address violation. Declaring a `char *` doesn't necessarily provide you with any storage to write into

Code after:

    if (num_digits >= max_digits) // Consume extra decimal digits.
      while (isdigit_ascii(*p))
        ++p;

    exponent -= num_decimals;
  }

  if (num_digits == 0) {
    *error = ERANGE;
    return 0.0;
  }

  // Correct for sign.
  if (negative)
    number = -number;

  // Process an exponent string.
  if (toupper_ascii(*p) == toupper_ascii(sci)) {
    if (maybe_int != NULL)
      *maybe_int = 0;

    // move past scientific notation
    p++;

    char *tmp_ptr;
    long int n = strtol(p, &tmp_ptr, 10);

    if (errno == ERANGE || checked_add(exponent, n, &exponent)) {
      errno = 0;
      exponent = n;
    }

    // If no digits after the 'e'/'E', un-consume it.
    if (tmp_ptr == p)
      p--;
    else
      p = tmp_ptr;
  }

  if (exponent > 308) {
    number = number == 0 ? 0 : number < 0 ? -HUGE_VAL : HUGE_VAL;
  } else if (exponent > 0) {
    number *= e[exponent];
  } else if (exponent < -308) { // Subnormal
    if (exponent < -616) {      // Prevent invalid array access.
      number = 0.;
    } else {
      number /= e[-308 - exponent];
      number /= e[308];
    }
Repository: pandas-dev/pandas
File: pandas/conftest.py (Python)

Code before:


# ----------------------------------------------------------------
# Misc
# ----------------------------------------------------------------
@pytest.fixture
def ip():
    """
    Get an instance of IPython.InteractiveShell.

    Will raise a skip if IPython is not installed.
    """
    pytest.importorskip("IPython", minversion="6.0.0")
    from IPython.core.interactiveshell import InteractiveShell

    # GH#35711 make sure sqlite history file handle is not leaked
    from traitlets.config import Config  # isort:skip

    c = Config()
    c.HistoryManager.hist_file = ":memory:"

    return InteractiveShell(config=c)


@pytest.fixture(autouse=True)
def mpl_cleanup(doctest_namespace):
    """
    Ensure Matplotlib is cleaned up around a test.

    Before a test is run:

    1) Set the backend to "template" to avoid requiring a GUI.

    After a test is run:

    1) Reset units registry
    2) Reset rc_context
    3) Close all figures

    See matplotlib/testing/decorators.py#L24.
    """
    mpl = pytest.importorskip("matplotlib")
    mpl_units = pytest.importorskip("matplotlib.units")
    plt = pytest.importorskip("matplotlib.pyplot")
    orig_units_registry = mpl_units.registry.copy()
    try:
        with mpl.rc_context():
            mpl.use("template")
            yield
    finally:
        mpl_units.registry.clear()

Review comment (suggestion):
Could we just change that matplotlib example instead (or add cleanup code in the doctest?). I don't think this fixture in particularly cheap to run for _every_ test

Code after:


# ----------------------------------------------------------------
# Misc
# ----------------------------------------------------------------
@pytest.fixture
def ip():
    """
    Get an instance of IPython.InteractiveShell.

    Will raise a skip if IPython is not installed.
    """
    pytest.importorskip("IPython", minversion="6.0.0")
    from IPython.core.interactiveshell import InteractiveShell

    # GH#35711 make sure sqlite history file handle is not leaked
    from traitlets.config import Config  # isort:skip

    c = Config()
    c.HistoryManager.hist_file = ":memory:"

    return InteractiveShell(config=c)


@pytest.fixture
def mpl_cleanup():
    """
    Ensure Matplotlib is cleaned up around a test.

    Before a test is run:

    1) Set the backend to "template" to avoid requiring a GUI.

    After a test is run:

    1) Reset units registry
    2) Reset rc_context
    3) Close all figures

    See matplotlib/testing/decorators.py#L24.
    """
    mpl = pytest.importorskip("matplotlib")
    mpl_units = pytest.importorskip("matplotlib.units")
    plt = pytest.importorskip("matplotlib.pyplot")
    orig_units_registry = mpl_units.registry.copy()
    try:
        with mpl.rc_context():
            mpl.use("template")
            yield
    finally:
        mpl_units.registry.clear()
Repository: pandas-dev/pandas
File: pandas/tests/indexing/interval/test_interval.py (Python)

Code before:
        )
        expected = Series([1, 6, 2, 8, 7], index=expected_index, name="value")
        tm.assert_series_equal(result, expected)

    @pytest.mark.xfail(WASM, reason="GH 23440")
    @pytest.mark.parametrize("base", [101, 1010])
    def test_reindex_behavior_with_interval_index(self, base):
        # GH 51826

        ser = Series(
            range(base),
            index=IntervalIndex.from_arrays(range(base), range(1, base + 1)),
        )
        expected_result = Series([np.nan, 0], index=[np.nan, 1.0], dtype=float)
        result = ser.reindex(index=[np.nan, 1.0])
        tm.assert_series_equal(result, expected_result)

    def test_multiindex_with_interval_index(self):
        # for GH#25298
        intIndex = IntervalIndex.from_arrays([1, 5, 8, 13, 16], [4, 9, 12, 17, 20])
        multiIndex = pd.MultiIndex.from_arrays([["a", "a", "b", "b", "c"], intIndex])
        data = [(1, 2), (3, 4), (5, 6), (7, 8), (9, 10)]
        df = DataFrame(data, index=multiIndex)
        series1 = df.loc[("b", 16)]
        series2 = df.loc["b"].loc[16]
        tm.assert_equal(series1.array, series2.array)


Review comment (suggestion):
I suggest using the assertion function `tm.assert_series_equal `instead of `tm.assert_equal`.  
We usually define an `expected` Series to check that left and right Series are equal (see the tests above).

Code after:
        )
        expected = Series([1, 6, 2, 8, 7], index=expected_index, name="value")
        tm.assert_series_equal(result, expected)

    @pytest.mark.xfail(WASM, reason="GH 23440")
    @pytest.mark.parametrize("base", [101, 1010])
    def test_reindex_behavior_with_interval_index(self, base):
        # GH 51826

        ser = Series(
            range(base),
            index=IntervalIndex.from_arrays(range(base), range(1, base + 1)),
        )
        expected_result = Series([np.nan, 0], index=[np.nan, 1.0], dtype=float)
        result = ser.reindex(index=[np.nan, 1.0])
        tm.assert_series_equal(result, expected_result)

    def test_multiindex_with_interval_index(self):
        # for GH#25298
        intIndex = IntervalIndex.from_arrays([1, 5, 8, 13, 16], [4, 9, 12, 17, 20])
        multiIndex = pd.MultiIndex.from_arrays([["a", "a", "b", "b", "c"], intIndex])
        data = [(1, 2), (3, 4), (5, 6), (7, 8), (9, 10)]
        df = DataFrame(data, index=multiIndex)
        result = df.loc[("b", 16)]
        expected = Series([7, 8], name=("b", pd.Interval(13, 17, closed="right")))
        tm.assert_series_equal(result, expected)