Repository: modin-project/modin

Issue:
BUG: external implementations need ray, dask, or hdk installed to run as custom execution
### Modin version checks

- [X] I have checked that this issue has not already been reported.

- [X] I have confirmed this bug exists on the latest released version of Modin.

- [X] I have confirmed this bug exists on the main branch of Modin. (In order to do this you can follow [this guide](https://modin.readthedocs.io/en/stable/getting_started/installation.html#installing-from-the-github-master-branch).)


### Reproducible Example

```python
from modin import set_execution
from modin.config import StorageFormat, Engine
from modin.core.storage_formats.pandas.query_compiler import PandasQueryCompiler
from modin.core.execution.dispatching.factories import factories
from modin.core.execution.python.implementations.pandas_on_python.io import (  # noqa: E402
    PandasOnPythonIO,
)
import modin.pandas as pd


class MyCustomQueryCompiler(PandasQueryCompiler):
    def __init__(self, *args, **kwargs):
        print(
            "initializing MyCustomQueryCompiler! this query compiler is not part of modin."
        )
        super().__init__(*args, **kwargs)


class MyCustomIO(PandasOnPythonIO):
    query_compiler_cls = MyCustomQueryCompiler


class MyCustomFactory(factories.BaseFactory):
    @classmethod
    def prepare(cls):
        cls.io_cls = MyCustomIO


factories.CustomOnCustomFactory = MyCustomFactory
StorageFormat.add_option("Custom")
Engine.add_option("Custom")

set_execution("Custom", "Custom")

df = pd.DataFrame([[1, 2, 3], [3, 4, 5], [5, 6, 7]])
assert isinstance(df._query_compiler, MyCustomQueryCompiler)
```


### Issue Description

The examples shown here works in a fresh conda environment with `pip install modin[ray]`, but not in one with just `pip install modin`. From the stack trace you can see that setting an engine requires getting the default engine, which Modin can't find.

### Expected Behavior

Users should be able to run Modin on custom storage and engine without having to have ray, dask, or HDK installed.

### Error Logs

<details>

```python-traceback

---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
File ~/software_sources/modin/modin/config/pubsub.py:260, in Parameter.get(cls)
    259 try:
--> 260     raw = cls._get_raw_from_config()
    261 except KeyError:

File ~/software_sources/modin/modin/config/envvars.py:53, in EnvironmentVariable._get_raw_from_config(cls)
     52     raise TypeError("varname should not be None")
---> 53 return os.environ[cls.varname]

File ~/opt/anaconda3/envs/modin-dev-py38/lib/python3.8/os.py:675, in _Environ.__getitem__(self, key)
    673 except KeyError:
    674     # raise KeyError with the original key value
--> 675     raise KeyError(key) from None
    676 return self.decodevalue(value)

KeyError: 'MODIN_ENGINE'

During handling of the above exception, another exception occurred:

ImportError                               Traceback (most recent call last)
Input In [1], in <cell line: 33>()
     30 StorageFormat.add_option("Custom")
     31 Engine.add_option("Custom")
---> 33 set_execution("Custom", "Custom")
     35 df = pd.DataFrame([[1, 2, 3], [3, 4, 5], [5, 6, 7]])
     36 assert isinstance(df._query_compiler, MyCustomQueryCompiler)

File ~/software_sources/modin/modin/__init__.py:60, in set_execution(engine, storage_format)
     58 # defer callbacks until both entities are set
     59 if engine is not None:
---> 60     old_engine = Engine._put_nocallback(engine)
     61 if storage_format is not None:
     62     old_storage_format = StorageFormat._put_nocallback(storage_format)

File ~/software_sources/modin/modin/config/pubsub.py:323, in Parameter._put_nocallback(cls, value)
    321     raise ValueError(f"Unsupported value: {value}")
    322 value = _TYPE_PARAMS[cls.type].normalize(value)
--> 323 oldvalue, cls._value = cls.get(), value
    324 return oldvalue

File ~/software_sources/modin/modin/config/pubsub.py:262, in Parameter.get(cls)
    260     raw = cls._get_raw_from_config()
    261 except KeyError:
--> 262     cls._value = cls._get_default()
    263     cls._value_source = ValueSource.DEFAULT
    264 else:

File ~/software_sources/modin/modin/config/envvars.py:154, in Engine._get_default(cls)
    148         raise ImportError(
    149             "Please `pip install unidist[mpi]` to install compatible unidist on MPI "
    150             + "version "
    151             + f"(>={MIN_UNIDIST_VERSION})."
    152         )
    153     return "Unidist"
--> 154 raise ImportError(
    155     "Please refer to installation documentation page to install an engine"
    156 )

ImportError: Please refer to installation documentation page to install an engine

```

</details>


### Installed Versions

Note I had to install dask for `show_versions` to work, whereas in the original environment I didn't have dask.

<details>
UserWarning: Setuptools is replacing distutils.

INSTALLED VERSIONS
------------------
commit           : 68b1f86b00e2492107168c1ad2f56d6277edce76
python           : 3.8.13.final.0
python-bits      : 64
OS               : Darwin
OS-release       : 21.5.0
Version          : Darwin Kernel Version 21.5.0: Tue Apr 26 21:08:22 PDT 2022; root:xnu-8020.121.3~4/RELEASE_X86_64
machine          : x86_64
processor        : i386
byteorder        : little
LC_ALL           : None
LANG             : en_US.UTF-8
LOCALE           : en_US.UTF-8

Modin dependencies
------------------
modin            : 0.17.0+33.g68b1f86b
ray              : None
dask             : 2022.11.1
distributed      : None
hdk              : None

pandas dependencies
-------------------
pandas           : 1.5.2
numpy            : 1.23.1
pytz             : 2022.1
dateutil         : 2.8.2
setuptools       : 61.2.0
pip              : 22.1.2
Cython           : None
pytest           : 7.1.2
hypothesis       : None
sphinx           : 5.1.1
blosc            : None
feather          : 0.4.1
xlsxwriter       : None
lxml.etree       : 4.9.1
html5lib         : None
pymysql          : None
psycopg2         : None
jinja2           : 3.1.2
IPython          : 8.4.0
pandas_datareader: None
bs4              : 4.11.1
bottleneck       : None
brotli           : None
fastparquet      : 2022.11.0
fsspec           : 2022.7.1
gcsfs            : None
matplotlib       : 3.5.2
numba            : None
numexpr          : 2.8.3
odfpy            : None
openpyxl         : 3.0.10
pandas_gbq       : 0.17.7
pyarrow          : 8.0.0
pyreadstat       : None
pyxlsb           : None
s3fs             : 2022.7.1
scipy            : 1.9.0
snappy           : None
sqlalchemy       : 1.4.39
tables           : 3.7.0
tabulate         : 0.9.0
xarray           : 2022.6.0
xlrd             : 2.0.1
xlwt             : None
zstandard        : None
tzdata           : None

</details>



Discussion:
cc @vnlitvinov just FYI

Resolving patch:
diff --git a/modin/config/envvars.py b/modin/config/envvars.py
--- a/modin/config/envvars.py
+++ b/modin/config/envvars.py
@@ -83,6 +83,8 @@ class Engine(EnvironmentVariable, type=str):
         "Python",
     }  # engines that don't require initialization, useful for unit tests
 
+    has_custom_engine = False
+
     @classmethod
     def _get_default(cls) -> str:
         """
@@ -98,7 +100,9 @@ def _get_default(cls) -> str:
             MIN_UNIDIST_VERSION,
         )
 
-        if IsDebug.get():
+        # If there's a custom engine, we don't need to check for any engine
+        # dependencies. Return the default "Python" engine.
+        if IsDebug.get() or cls.has_custom_engine:
             return "Python"
         try:
             import ray
@@ -160,6 +164,7 @@ def _get_default(cls) -> str:
     def add_option(cls, choice: Any) -> Any:
         choice = super().add_option(choice)
         cls.NOINIT_ENGINES.add(choice)
+        cls.has_custom_engine = True
         return choice
 
 
diff --git a/modin/core/execution/dispatching/factories/dispatcher.py b/modin/core/execution/dispatching/factories/dispatcher.py
--- a/modin/core/execution/dispatching/factories/dispatcher.py
+++ b/modin/core/execution/dispatching/factories/dispatcher.py
@@ -109,7 +109,9 @@ class FactoryDispatcher(object):
     @classmethod
     def get_factory(cls) -> factories.BaseFactory:
         """Get current factory."""
-        # mostly for testing
+        if cls.__factory is None:
+            Engine.subscribe(cls._update_factory)
+            StorageFormat.subscribe(cls._update_factory)
         return cls.__factory
 
     @classmethod
@@ -155,152 +157,148 @@ def _update_factory(cls, _):
     @classmethod
     @_inherit_docstrings(factories.BaseFactory._from_pandas)
     def from_pandas(cls, df):
-        return cls.__factory._from_pandas(df)
+        return cls.get_factory()._from_pandas(df)
 
     @classmethod
     @_inherit_docstrings(factories.BaseFactory._from_arrow)
     def from_arrow(cls, at):
-        return cls.__factory._from_arrow(at)
+        return cls.get_factory()._from_arrow(at)
 
     @classmethod
     @_inherit_docstrings(factories.BaseFactory._from_non_pandas)
     def from_non_pandas(cls, *args, **kwargs):
-        return cls.__factory._from_non_pandas(*args, **kwargs)
+        return cls.get_factory()._from_non_pandas(*args, **kwargs)
 
     @classmethod
     @_inherit_docstrings(factories.BaseFactory._from_dataframe)
     def from_dataframe(cls, *args, **kwargs):
-        return cls.__factory._from_dataframe(*args, **kwargs)
+        return cls.get_factory()._from_dataframe(*args, **kwargs)
 
     @classmethod
     @_inherit_docstrings(factories.BaseFactory._read_parquet)
     def read_parquet(cls, **kwargs):
-        return cls.__factory._read_parquet(**kwargs)
+        return cls.get_factory()._read_parquet(**kwargs)
 
     @classmethod
     @_inherit_docstrings(factories.BaseFactory._read_csv)
     def read_csv(cls, **kwargs):
-        return cls.__factory._read_csv(**kwargs)
+        return cls.get_factory()._read_csv(**kwargs)
 
     @classmethod
     @_inherit_docstrings(factories.ExperimentalPandasOnRayFactory._read_csv_glob)
     def read_csv_glob(cls, **kwargs):
-        return cls.__factory._read_csv_glob(**kwargs)
+        return cls.get_factory()._read_csv_glob(**kwargs)
 
     @classmethod
     @_inherit_docstrings(
         factories.ExperimentalPandasOnRayFactory._read_pickle_distributed
     )
     def read_pickle_distributed(cls, **kwargs):
-        return cls.__factory._read_pickle_distributed(**kwargs)
+        return cls.get_factory()._read_pickle_distributed(**kwargs)
 
     @classmethod
     @_inherit_docstrings(factories.BaseFactory._read_json)
     def read_json(cls, **kwargs):
-        return cls.__factory._read_json(**kwargs)
+        return cls.get_factory()._read_json(**kwargs)
 
     @classmethod
     @_inherit_docstrings(factories.BaseFactory._read_gbq)
     def read_gbq(cls, **kwargs):
-        return cls.__factory._read_gbq(**kwargs)
+        return cls.get_factory()._read_gbq(**kwargs)
 
     @classmethod
     @_inherit_docstrings(factories.BaseFactory._read_html)
     def read_html(cls, **kwargs):
-        return cls.__factory._read_html(**kwargs)
+        return cls.get_factory()._read_html(**kwargs)
 
     @classmethod
     @_inherit_docstrings(factories.BaseFactory._read_clipboard)
     def read_clipboard(cls, **kwargs):
-        return cls.__factory._read_clipboard(**kwargs)
+        return cls.get_factory()._read_clipboard(**kwargs)
 
     @classmethod
     @_inherit_docstrings(factories.BaseFactory._read_excel)
     def read_excel(cls, **kwargs):
-        return cls.__factory._read_excel(**kwargs)
+        return cls.get_factory()._read_excel(**kwargs)
 
     @classmethod
     @_inherit_docstrings(factories.BaseFactory._read_hdf)
     def read_hdf(cls, **kwargs):
-        return cls.__factory._read_hdf(**kwargs)
+        return cls.get_factory()._read_hdf(**kwargs)
 
     @classmethod
     @_inherit_docstrings(factories.BaseFactory._read_feather)
     def read_feather(cls, **kwargs):
-        return cls.__factory._read_feather(**kwargs)
+        return cls.get_factory()._read_feather(**kwargs)
 
     @classmethod
     @_inherit_docstrings(factories.BaseFactory._read_stata)
     def read_stata(cls, **kwargs):
-        return cls.__factory._read_stata(**kwargs)
+        return cls.get_factory()._read_stata(**kwargs)
 
     @classmethod
     @_inherit_docstrings(factories.BaseFactory._read_sas)
     def read_sas(cls, **kwargs):  # pragma: no cover
-        return cls.__factory._read_sas(**kwargs)
+        return cls.get_factory()._read_sas(**kwargs)
 
     @classmethod
     @_inherit_docstrings(factories.BaseFactory._read_pickle)
     def read_pickle(cls, **kwargs):
-        return cls.__factory._read_pickle(**kwargs)
+        return cls.get_factory()._read_pickle(**kwargs)
 
     @classmethod
     @_inherit_docstrings(factories.BaseFactory._read_sql)
     def read_sql(cls, **kwargs):
-        return cls.__factory._read_sql(**kwargs)
+        return cls.get_factory()._read_sql(**kwargs)
 
     @classmethod
     @_inherit_docstrings(factories.BaseFactory._read_fwf)
     def read_fwf(cls, **kwargs):
-        return cls.__factory._read_fwf(**kwargs)
+        return cls.get_factory()._read_fwf(**kwargs)
 
     @classmethod
     @_inherit_docstrings(factories.BaseFactory._read_sql_table)
     def read_sql_table(cls, **kwargs):
-        return cls.__factory._read_sql_table(**kwargs)
+        return cls.get_factory()._read_sql_table(**kwargs)
 
     @classmethod
     @_inherit_docstrings(factories.BaseFactory._read_sql_query)
     def read_sql_query(cls, **kwargs):
-        return cls.__factory._read_sql_query(**kwargs)
+        return cls.get_factory()._read_sql_query(**kwargs)
 
     @classmethod
     @_inherit_docstrings(factories.BaseFactory._read_spss)
     def read_spss(cls, **kwargs):
-        return cls.__factory._read_spss(**kwargs)
+        return cls.get_factory()._read_spss(**kwargs)
 
     @classmethod
     @_inherit_docstrings(factories.BaseFactory._to_sql)
     def to_sql(cls, *args, **kwargs):
-        return cls.__factory._to_sql(*args, **kwargs)
+        return cls.get_factory()._to_sql(*args, **kwargs)
 
     @classmethod
     @_inherit_docstrings(factories.BaseFactory._to_pickle)
     def to_pickle(cls, *args, **kwargs):
-        return cls.__factory._to_pickle(*args, **kwargs)
+        return cls.get_factory()._to_pickle(*args, **kwargs)
 
     @classmethod
     @_inherit_docstrings(
         factories.ExperimentalPandasOnRayFactory._to_pickle_distributed
     )
     def to_pickle_distributed(cls, *args, **kwargs):
-        return cls.__factory._to_pickle_distributed(*args, **kwargs)
+        return cls.get_factory()._to_pickle_distributed(*args, **kwargs)
 
     @classmethod
     @_inherit_docstrings(factories.ExperimentalPandasOnRayFactory._read_custom_text)
     def read_custom_text(cls, **kwargs):
-        return cls.__factory._read_custom_text(**kwargs)
+        return cls.get_factory()._read_custom_text(**kwargs)
 
     @classmethod
     @_inherit_docstrings(factories.BaseFactory._to_csv)
     def to_csv(cls, *args, **kwargs):
-        return cls.__factory._to_csv(*args, **kwargs)
+        return cls.get_factory()._to_csv(*args, **kwargs)
 
     @classmethod
     @_inherit_docstrings(factories.BaseFactory._to_parquet)
     def to_parquet(cls, *args, **kwargs):
-        return cls.__factory._to_parquet(*args, **kwargs)
-
-
-Engine.subscribe(FactoryDispatcher._update_factory)
-StorageFormat.subscribe(FactoryDispatcher._update_factory)
+        return cls.get_factory()._to_parquet(*args, **kwargs)


Test patch: