Repository: modin-project/modin (2025-05-02 ~ 2025-05-07)

problem_statement: Stop creating an empty dataframe on every nullary operation
just do this once for each backend: https://github.com/modin-project/modin/blob/b5bd8cd4fd600b4708692bc0b5d007a132e5dc4b/modin/core/storage_formats/pandas/query_compiler_caster.py#L822

Creating an empty dataframe can be expensive.
date: 2025-05-02
patch: diff --git a/modin/core/storage_formats/pandas/query_compiler_caster.py b/modin/core/storage_formats/pandas/query_compiler_caster.py
index 3027d238..6d5ae13d 100644
--- a/modin/core/storage_formats/pandas/query_compiler_caster.py
+++ b/modin/core/storage_formats/pandas/query_compiler_caster.py
@@ -88,6 +88,25 @@ _CLASS_AND_BACKEND_TO_PRE_OP_SWITCH_METHODS: _AUTO_SWITCH_CLASS = _AUTO_SWITCH_C
 )
 
 
+def _get_empty_qc_for_default_backend() -> BaseQueryCompiler:
+    """
+    Get an empty query compiler for the default backend.
+
+    Returns
+    -------
+    BaseQueryCompiler
+        An empty query compiler for the default backend.
+    """
+    from modin.core.execution.dispatching.factories.dispatcher import FactoryDispatcher
+
+    return FactoryDispatcher.get_factory().io_cls.from_pandas(pandas.DataFrame())
+
+
+_BACKEND_TO_EMPTY_QC: defaultdict[str, BaseQueryCompiler] = defaultdict(
+    _get_empty_qc_for_default_backend
+)
+
+
 class QueryCompilerCaster(ABC):
     """Cast all query compiler arguments of the member function to current query compiler."""
 
@@ -793,16 +812,11 @@ def wrap_function_in_argument_caster(
         #    backends, raise a ValueError.
 
         if len(input_query_compilers) == 0:
-            # For nullary functions, we need to create a dummy query compiler
-            # to calculate the cost of switching backends.
-            from modin.core.execution.dispatching.factories.dispatcher import (
-                FactoryDispatcher,
-            )
-
-            input_qc_for_pre_op_switch = (
-                FactoryDispatcher.get_factory().io_cls.from_pandas(pandas.DataFrame())
-            )
             input_backend = Backend.get()
+            # For nullary functions, we need to create a dummy query compiler
+            # to calculate the cost of switching backends. We should only
+            # create the dummy query compiler once per backend.
+            input_qc_for_pre_op_switch = _BACKEND_TO_EMPTY_QC[input_backend]
         else:
             input_qc_for_pre_op_switch = input_query_compilers[0]
             input_backend = input_qc_for_pre_op_switch.get_backend()

test_patch: diff --git a/modin/tests/pandas/native_df_interoperability/test_compiler_caster.py b/modin/tests/pandas/native_df_interoperability/test_compiler_caster.py
index 688a5685..cfef3ae7 100644
--- a/modin/tests/pandas/native_df_interoperability/test_compiler_caster.py
+++ b/modin/tests/pandas/native_df_interoperability/test_compiler_caster.py
@@ -1257,3 +1257,17 @@ def test_concat_with_pin(pin_backends, expected_backend):
             df_equals(
                 result, pandas.concat([pandas.DataFrame([1] * 10)] * len(pin_backends))
             )
+
+
+def test_second_init_only_calls_from_pandas_once_github_issue_7559():
+    with config_context(Backend="Big_Data_Cloud"):
+        # Create a dataframe once first so that we can initialize the dummy
+        # query compiler for the Big_Data_Cloud backend.
+        pd.DataFrame([1])
+        with mock.patch.object(
+            factories.Big_Data_CloudOnNativeFactory.io_cls.query_compiler_cls,
+            "from_pandas",
+            wraps=factories.Big_Data_CloudOnNativeFactory.io_cls.query_compiler_cls.from_pandas,
+        ) as mock_from_pandas:
+            pd.DataFrame([1])
+            mock_from_pandas.assert_called_once()


----

problem_statement: Need the ability to configure NativeQueryCompiler with environment variables
We have several class variables in the native query compiler which should be configurable.
date: 2025-05-07
patch: diff --git a/modin/config/__init__.py b/modin/config/__init__.py
index 6a258733..aeaadd95 100644
--- a/modin/config/__init__.py
+++ b/modin/config/__init__.py
@@ -44,6 +44,8 @@ from modin.config.envvars import (
     MinPartitionSize,
     MinRowPartitionSize,
     ModinNumpy,
+    NativePandasMaxRows,
+    NativePandasTransferThreshold,
     NPartitions,
     PersistentPickle,
     ProgressBar,
@@ -85,6 +87,9 @@ __all__ = [
     "LazyExecution",
     # Dask specific
     "DaskThreadsPerWorker",
+    # Native Pandas Specific
+    "NativePandasMaxRows",
+    "NativePandasTransferThreshold",
     # Partitioning
     "NPartitions",
     "MinPartitionSize",
diff --git a/modin/config/envvars.py b/modin/config/envvars.py
index 97c99f40..27fbda6e 100644
--- a/modin/config/envvars.py
+++ b/modin/config/envvars.py
@@ -1305,6 +1305,27 @@ class DaskThreadsPerWorker(EnvironmentVariable, type=int):
     default = 1
 
 
+class NativePandasMaxRows(EnvironmentVariable, type=int):
+    """
+    Maximum number of rows which can be processed using local, native, pandas.
+    """
+
+    varname = "MODIN_NATIVE_MAX_ROWS"
+    default = 10_000_000
+
+
+class NativePandasTransferThreshold(EnvironmentVariable, type=int):
+    """
+    Targeted max number of dataframe rows which should be transferred between engines.
+
+    This is often the same value as MODIN_NATIVE_MAX_ROWS but it can be independently
+    set to change how transfer costs are considered.
+    """
+
+    varname = "MODIN_NATIVE_MAX_XFER_ROWS"
+    default = 10_000_000
+
+
 class DynamicPartitioning(EnvironmentVariable, type=bool):
     """
     Set to true to use Modin's dynamic-partitioning implementation where possible.
diff --git a/modin/core/storage_formats/base/query_compiler.py b/modin/core/storage_formats/base/query_compiler.py
index a1375677..0ac79bb2 100644
--- a/modin/core/storage_formats/base/query_compiler.py
+++ b/modin/core/storage_formats/base/query_compiler.py
@@ -356,13 +356,13 @@ class BaseQueryCompiler(
         """
         if isinstance(self, other_qc_type):
             return QCCoercionCost.COST_ZERO
-        if self._TRANSFER_THRESHOLD <= 0:
+        if self.__class__._transfer_threshold() <= 0:
             return QCCoercionCost.COST_ZERO
         cost = int(
             (
                 QCCoercionCost.COST_IMPOSSIBLE
                 * self._max_shape()[0]
-                / self._TRANSFER_THRESHOLD
+                / self.__class__._transfer_threshold()
             )
         )
         if cost > QCCoercionCost.COST_IMPOSSIBLE:
@@ -447,7 +447,7 @@ class BaseQueryCompiler(
         return self._stay_cost_rows(
             self._max_shape()[0],
             self._OPERATION_PER_ROW_OVERHEAD,
-            self._MAX_SIZE_THIS_ENGINE_CAN_HANDLE,
+            self.__class__._engine_max_size(),
             self._OPERATION_INITIALIZATION_OVERHEAD,
         )
 
@@ -498,10 +498,18 @@ class BaseQueryCompiler(
         return cls._stay_cost_rows(
             row_count,
             cls._OPERATION_PER_ROW_OVERHEAD,
-            cls._MAX_SIZE_THIS_ENGINE_CAN_HANDLE,
+            cls._engine_max_size(),
             cls._OPERATION_INITIALIZATION_OVERHEAD,
         )
 
+    @classmethod
+    def _engine_max_size(cls):
+        return cls._MAX_SIZE_THIS_ENGINE_CAN_HANDLE
+
+    @classmethod
+    def _transfer_threshold(cls):
+        return cls._TRANSFER_THRESHOLD
+
     @disable_logging
     def max_cost(self) -> int:
         """
diff --git a/modin/core/storage_formats/pandas/native_query_compiler.py b/modin/core/storage_formats/pandas/native_query_compiler.py
index 74f8b694..ad17a115 100644
--- a/modin/core/storage_formats/pandas/native_query_compiler.py
+++ b/modin/core/storage_formats/pandas/native_query_compiler.py
@@ -23,12 +23,11 @@ from typing import Optional
 import pandas
 from pandas.core.dtypes.common import is_scalar
 
+from modin.config.envvars import NativePandasMaxRows, NativePandasTransferThreshold
 from modin.core.dataframe.base.interchange.dataframe_protocol.dataframe import (
     ProtocolDataframe,
 )
-from modin.core.storage_formats.base.query_compiler import (
-    BaseQueryCompiler,
-)
+from modin.core.storage_formats.base.query_compiler import BaseQueryCompiler
 from modin.utils import _inherit_docstrings
 
 _NO_REPARTITION_ON_NATIVE_EXECUTION_EXCEPTION_MESSAGE = (
@@ -92,10 +91,8 @@ class NativeQueryCompiler(BaseQueryCompiler):
         The pandas frame to query with the compiled queries.
     """
 
-    _MAX_SIZE_THIS_ENGINE_CAN_HANDLE = 10_000_000
     _OPERATION_INITIALIZATION_OVERHEAD = 0
     _OPERATION_PER_ROW_OVERHEAD = 0
-    _TRANSFER_THRESHOLD = 10_000_000
 
     _modin_frame: pandas.DataFrame
     _should_warn_on_default_to_pandas: bool = False
@@ -212,8 +209,21 @@ class NativeQueryCompiler(BaseQueryCompiler):
     def finalize(self):
         return
 
-    # Dataframe interchange protocol
+    @classmethod
+    def _engine_max_size(cls):
+        # do not return the custom configuration for sub-classes
+        if cls == NativeQueryCompiler:
+            return NativePandasMaxRows.get()
+        return cls._MAX_SIZE_THIS_ENGINE_CAN_HANDLE
 
+    @classmethod
+    def _transfer_threshold(cls):
+        # do not return the custom configuration for sub-classes
+        if cls == NativeQueryCompiler:
+            return NativePandasTransferThreshold.get()
+        return cls._TRANSFER_THRESHOLD
+
+    # Dataframe interchange protocol
     def to_interchange_dataframe(
         self, nan_as_null: bool = False, allow_copy: bool = True
     ):

test_patch: diff --git a/modin/tests/pandas/native_df_interoperability/test_compiler_caster.py b/modin/tests/pandas/native_df_interoperability/test_compiler_caster.py
index cfef3ae7..c5bb52d5 100644
--- a/modin/tests/pandas/native_df_interoperability/test_compiler_caster.py
+++ b/modin/tests/pandas/native_df_interoperability/test_compiler_caster.py
@@ -24,7 +24,13 @@ from pytest import param
 
 import modin.pandas as pd
 from modin.config import context as config_context
-from modin.config.envvars import Backend, Engine, Execution
+from modin.config.envvars import (
+    Backend,
+    Engine,
+    Execution,
+    NativePandasMaxRows,
+    NativePandasTransferThreshold,
+)
 from modin.core.execution.dispatching.factories import factories
 from modin.core.execution.dispatching.factories.factories import BaseFactory
 from modin.core.io.io import BaseIO
@@ -1271,3 +1277,34 @@ def test_second_init_only_calls_from_pandas_once_github_issue_7559():
         ) as mock_from_pandas:
             pd.DataFrame([1])
             mock_from_pandas.assert_called_once()
+
+
+def test_native_config():
+    qc = NativeQueryCompiler(pandas.DataFrame([0, 1, 2]))
+
+    # Native Query Compiler gets a special configuration
+    assert qc._TRANSFER_THRESHOLD == 0
+    assert qc._transfer_threshold() == NativePandasTransferThreshold.get()
+    assert qc._MAX_SIZE_THIS_ENGINE_CAN_HANDLE == 1
+    assert qc._engine_max_size() == NativePandasMaxRows.get()
+
+    oldmax = qc._engine_max_size()
+    oldthresh = qc._transfer_threshold()
+
+    with config_context(NativePandasMaxRows=123, NativePandasTransferThreshold=321):
+        qc2 = NativeQueryCompiler(pandas.DataFrame([0, 1, 2]))
+        assert qc2._transfer_threshold() == 321
+        assert qc2._engine_max_size() == 123
+        assert qc._engine_max_size() == 123
+        assert qc._transfer_threshold() == 321
+
+        # sub class configuration is unchanged
+        class AQC(NativeQueryCompiler):
+            pass
+
+        subqc = AQC(pandas.DataFrame([0, 1, 2]))
+        assert subqc._TRANSFER_THRESHOLD == 0
+        assert subqc._MAX_SIZE_THIS_ENGINE_CAN_HANDLE == 1
+
+    assert qc._engine_max_size() == oldmax
+    assert qc._transfer_threshold() == oldthresh


----

problem_statement: BUG: getting extension property that should raise AttributeError instead gets the property itself
### Modin version checks

- [x] I have checked that this issue has not already been reported.

- [x] I have confirmed this bug exists on the latest released version of Modin.

- [x] I have confirmed this bug exists on the main branch of Modin. (In order to do this you can follow [this guide](https://modin.readthedocs.io/en/stable/getting_started/installation.html#installing-from-the-github-main-branch).)


### Reproducible Example

```python
from modin.pandas.api.extensions import register_base_accessor
import modin.pandas as pd
import pytest

def _get_property(self):
    raise AttributeError


register_base_accessor('new_property')(property(_get_property))
df = pd.DataFrame()
df.new_property

with pytest.raises(AttributeError):
    df.new_property
```

### Issue Description

if getting an extension property should raise `AttributeError`, modin returns the `property` object itself instead of raise the same error.

I think the lack of a `__get__` call here is the reason: https://github.com/modin-project/modin/blob/837fb954b7e05555652546e8bd68b63ddcf65e09/modin/pandas/base.py#L4694

we should delete `_getattr__from_extension_impl`. `__getattribute__` can take care of getting extension properties.

### Expected Behavior

should raise `AttributeError`

### Error Logs

N/A

### Installed Versions

<details>

INSTALLED VERSIONS
------------------
commit                : 837fb954b7e05555652546e8bd68b63ddcf65e09
python                : 3.9.21
python-bits           : 64
OS                    : Darwin
OS-release            : 24.4.0
Version               : Darwin Kernel Version 24.4.0: Fri Apr 11 18:33:39 PDT 2025; root:xnu-11417.101.15~117/RELEASE_ARM64_T6020
machine               : arm64
processor             : arm
byteorder             : little
LC_ALL                : None
LANG                  : en_US.UTF-8
LOCALE                : en_US.UTF-8

Modin dependencies
------------------
modin                 : 0.32.0+58.g837fb954.dirty
ray                   : 2.44.1
dask                  : 2024.8.0
distributed           : 2024.8.0

pandas dependencies
-------------------
pandas                : 2.2.3
numpy                 : 2.0.2
pytz                  : 2025.2
dateutil              : 2.9.0.post0
pip                   : 24.2
Cython                : None
sphinx                : 7.4.7
IPython               : 8.18.1
adbc-driver-postgresql: None
adbc-driver-sqlite    : None
bs4                   : 4.13.4
blosc                 : None
bottleneck            : None
dataframe-api-compat  : None
fastparquet           : 2024.11.0
fsspec                : 2025.3.2
html5lib              : None
hypothesis            : None
gcsfs                 : None
jinja2                : 3.1.6
lxml.etree            : 5.3.2
matplotlib            : 3.9.4
numba                 : None
numexpr               : 2.10.2
odfpy                 : None
openpyxl              : 3.1.5
pandas_gbq            : 0.28.0
psycopg2              : 2.9.10
pymysql               : None
pyarrow               : 19.0.1
pyreadstat            : None
pytest                : 8.3.5
python-calamine       : None
pyxlsb                : None
s3fs                  : 2025.3.2
scipy                 : 1.13.1
sqlalchemy            : 2.0.40
tables                : N/A
tabulate              : 0.9.0
xarray                : 2024.7.0
xlrd                  : 2.0.1
xlsxwriter            : None
zstandard             : None
tzdata                : 2025.2
qtpy                  : None
pyqt5                 : None

</details>

date: 2025-05-07
patch: diff --git a/modin/core/storage_formats/pandas/query_compiler_caster.py b/modin/core/storage_formats/pandas/query_compiler_caster.py
index 5ddd2802..96df688f 100644
--- a/modin/core/storage_formats/pandas/query_compiler_caster.py
+++ b/modin/core/storage_formats/pandas/query_compiler_caster.py
@@ -59,7 +59,6 @@ _NON_EXTENDABLE_ATTRIBUTES = {
     "__delattr__",
     "__getattr__",
     "_getattribute__from_extension_impl",
-    "_getattr__from_extension_impl",
     "get_backend",
     "move_to",
     "_update_inplace",
diff --git a/modin/pandas/base.py b/modin/pandas/base.py
index 29fe3aef..a7f9f8f8 100644
--- a/modin/pandas/base.py
+++ b/modin/pandas/base.py
@@ -110,7 +110,6 @@ _EXTENSION_NO_LOOKUP = {
     "_query_compiler",
     "get_backend",
     "_getattribute__from_extension_impl",
-    "_getattr__from_extension_impl",
     "_get_query_compiler",
     "set_backend",
     "_pinned",
@@ -4376,29 +4375,6 @@ class BasePandasDataset(QueryCompilerCaster, ClassLogger):
                 return default_handler
         return attr
 
-    @disable_logging
-    def __getattr__(self, item) -> Any:
-        """
-        Return attribute from this `BasePandasDataset`.
-
-        Parameters
-        ----------
-        item : str
-            Item to get.
-
-        Returns
-        -------
-        Any
-            The attribute from this `BasePandasDataset`.
-        """
-        # NOTE that to get an attribute, python calls __getattribute__() first and
-        # then falls back to __getattr__() if the former raises an AttributeError.
-        if item not in _EXTENSION_NO_LOOKUP:
-            extension = self._getattr__from_extension_impl(item, __class__._extensions)
-            if extension is not sentinel:
-                return extension
-        return object.__getattribute__(self, item)
-
     def __array_ufunc__(
         self, ufunc: np.ufunc, method: str, *inputs: Any, **kwargs: Any
     ) -> DataFrame | Series | Any:
@@ -4664,36 +4640,6 @@ class BasePandasDataset(QueryCompilerCaster, ClassLogger):
             )
         return sentinel
 
-    @disable_logging
-    def _getattr__from_extension_impl(self, item, extensions: EXTENSION_DICT_TYPE):
-        """
-        __getattr__() an extension with the given name from the given set of extensions.
-
-        Implement __getattr__() for extensions. python falls back to
-        __getattr__() if __getattribute__() raises an AttributeError.
-
-        Parameters
-        ----------
-        item : str
-            The name of the attribute to get.
-        extensions : EXTENSION_DICT_TYPE
-            The set of extensions.
-
-        Returns
-        -------
-        Any
-            The attribute from the extension, or `sentinel` if the attribute is
-            not found.
-        """
-        extension = self._get_extension(item, extensions)
-        if extension is not sentinel:
-            # We need to implement callable extensions before we fall back
-            # to __getattr__(), because they need to dispatch to the
-            # appropriate backend.
-            assert not callable(extension)
-            return extension
-        return sentinel
-
     @disable_logging
     @_inherit_docstrings(QueryCompilerCaster._get_query_compiler)
     def _get_query_compiler(self):
diff --git a/modin/pandas/dataframe.py b/modin/pandas/dataframe.py
index e9e39fe1..755ec266 100644
--- a/modin/pandas/dataframe.py
+++ b/modin/pandas/dataframe.py
@@ -2644,11 +2644,6 @@ class DataFrame(BasePandasDataset):
         """
         # NOTE that to get an attribute, python calls __getattribute__() first and
         # then falls back to __getattr__() if the former raises an AttributeError.
-
-        if key not in _EXTENSION_NO_LOOKUP:
-            extension = self._getattr__from_extension_impl(key, __class__._extensions)
-            if extension is not sentinel:
-                return extension
         try:
             return super().__getattr__(key)
         except AttributeError as err:
diff --git a/modin/pandas/series.py b/modin/pandas/series.py
index 139f3f08..c5d08df9 100644
--- a/modin/pandas/series.py
+++ b/modin/pandas/series.py
@@ -387,10 +387,6 @@ class Series(BasePandasDataset):
         """
         # NOTE that to get an attribute, python calls __getattribute__() first and
         # then falls back to __getattr__() if the former raises an AttributeError.
-        if key not in _EXTENSION_NO_LOOKUP:
-            extension = self._getattr__from_extension_impl(key, __class__._extensions)
-            if extension is not sentinel:
-                return extension
         try:
             return super().__getattr__(key)
         except AttributeError as err:

test_patch: diff --git a/modin/tests/pandas/extensions/test_base_extensions.py b/modin/tests/pandas/extensions/test_base_extensions.py
index e45438c8..50ec0c1f 100644
--- a/modin/tests/pandas/extensions/test_base_extensions.py
+++ b/modin/tests/pandas/extensions/test_base_extensions.py
@@ -217,13 +217,23 @@ class TestProperty:
         )
 
         modin_object = data_class({"a": [1, 2, 3], "b": [4, 5, 6]}).set_backend(backend)
-        assert hasattr(modin_object, public_property_name)
         setattr(modin_object, public_property_name, "value")
         assert getattr(modin_object, public_property_name) == "value"
         delattr(modin_object, public_property_name)
         # check that the deletion works.
         assert not hasattr(modin_object, private_property_name)
 
+    def test_get_property_that_raises_attribute_error_on_get_modin_issue_7562(
+        self, data_class
+    ):
+        def get_property(self):
+            raise AttributeError
+
+        register_base_accessor(name="extension_property")(property(fget=get_property))
+        modin_object = data_class()
+        with pytest.raises(AttributeError):
+            getattr(modin_object, "extension_property")
+
     def test_non_settable_extension_property(self, Backend1, data_class):
         modin_object = data_class([0])
         property_name = "property_name"