Repository: modin-project/modin

Issue:
Performance regression relative to 0.7.4 release
### System information
- **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**: Ubuntu 18.04
- **Modin version** (`modin.__version__`): 41d31119d6e6d81c8d9ea520899ce988f1fa8093
- **Python version**: 3.8.6
- **Code we can use to reproduce**:
<details>
  <summary>reproducer</summary>

```
import subprocess
from timeit import default_timer as timer
import json
import os

data_file = "/dataset/path/yellow_tripdata_2015-01.csv"
iterations_number = 3
report_file = "regression_report.txt"
report_metric_file = "regression_metric_report.txt"
report_modin_good = "modin_good_results.txt"
max_g2b_deviation = 0.8 # good/bad

def get_git_revision_hash(encoding="ascii"):
    return subprocess.check_output(['git', 'rev-parse', 'HEAD'], encoding=encoding).strip()

def measure(func, *args, **kw):
    t0 = timer()
    res = func(*args, **kw)
    t = timer() - t0
    return res, t

def q_read_csv(pd):
    res = pd.read_csv(data_file, parse_dates=["tpep_pickup_datetime", "tpep_dropoff_datetime"], quoting=3)
    repr(res)
    return res

def q_reductions(df):
    res = df.count()
    repr(res)
    return res

def q_map_operations(df):
    res = df.isnull()
    repr(res)
    return res

def q_appy(df):
    res = df["trip_distance"].apply(round)
    repr(res)
    return res

def q_add_column(df, col):
    df["rounded_trip_distance"] = col
    repr(df)
    return df

def q_groupby_agg(df):
    res = df.groupby(by="rounded_trip_distance").count()
    repr(res)
    return res

def bench(pd):
    results = {}
    df, results["t_read_csv"] = measure(q_read_csv, pd)
    _, results["t_reductions"] =  measure(q_reductions, df)
    _, results["t_map_operations"] = measure(q_map_operations, df)
    col_apply, results["t_apply"] = measure(q_appy, df)
    _, results["t_add_column"] = measure(q_add_column, df, col_apply)
    _, results["t_groupby_agg"] = measure(q_groupby_agg, df)

    return results

def bench_iterations(iterations=1):
    results = {}
    final_results = {}
    
    for i in range(1, iterations+1):
        import modin.pandas as pd
        pd.DEFAULT_NPARTITIONS = 4
        results[f"iteration_{i}"] = bench(pd)

    for t in results["iteration_1"].keys():
        values = []
        for it in results.keys():
            values.append(results[it][t])

        final_results[t] = min(values)

    return final_results

def make_report(result, report_file=report_file):
    with open(report_file, mode = 'a') as file:
        file.write(json.dumps(result) + "\n")

def metric(good_results: dict, bad_results: dict):
    coefficients = {}
    for t in good_results.keys():
        assert t in bad_results.keys()
        coefficients[t] = good_results[t] / bad_results[t]

    count = 0
    _sum = 0
    for coeff_key in coefficients:
        count += 1
        _sum += coefficients[coeff_key]

    mean = _sum / count
    good = mean > max_g2b_deviation
    coefficients["version"] = get_git_revision_hash()
    coefficients["mean"] = mean

    return coefficients, good

def eval_modin_bench(good_results):
    reults = bench_iterations(iterations=iterations_number)
    metrics, good = metric(good_results, reults)

    reults["version"] = get_git_revision_hash()

    make_report(reults, report_file=report_file)
    make_report(metrics, report_file=report_metric_file)
    print("reults: \n", reults)
    print("metrics: \n", metrics)

    return good

def remove_old_reports():
    for f in [report_file, report_metric_file]:
        if os.path.exists(f):
            os.remove(f)

def get_good_results(fname):
    with open(fname) as json_file:
        data = json.load(json_file)
    
    return data
    

if __name__ == "__main__":
    # uncomment to measure reference (good commit)
    # remove_old_reports()

    # reults_modin_good = bench_iterations(iterations=iterations_number)
    # print(reults_modin_good)
    # make_report(reults_modin_good, report_file=report_modin_good)

    good_results = get_good_results(report_modin_good)
    good = eval_modin_bench(good_results)
    print("good", good)
```


</details>
<!--
You can obtain the Modin version with

python -c "import modin; print(modin.__version__)"
-->

### Describe the problem
<!-- Describe the problem clearly here. -->
It was found that performance of some operations from [example notebook](https://github.com/modin-project/modin/blob/master/examples/tutorial/tutorial_notebooks/introduction/exercise_2.ipynb) degraded relative to 0.7.4 release. Measurement results are next (given relation of queries execution times t_0.7.4/t_master, numbers lower 1 corresponds to degradation from 0.7.4 release to current master)

read_csv 1.23 (reading from local csv)
reductions 0.98
map operations 0.62
apply operation 0.95
add column operation 0.23
groupby.agg operation 0.34

mean value for all queries 0.72

### Source code / logs
<!-- Include any logs or source code that would be helpful to diagnose the problem. If including tracebacks, please include the full traceback. Large logs and files should be attached. Try to provide a reproducible test case that is the bare minimum necessary to generate the problem. -->



Discussion:

Resolving patch:
diff --git a/modin/engines/base/frame/data.py b/modin/engines/base/frame/data.py
--- a/modin/engines/base/frame/data.py
+++ b/modin/engines/base/frame/data.py
@@ -777,9 +777,7 @@ def astype(self, col_dtypes):
         def astype_builder(df):
             return df.astype({k: v for k, v in col_dtypes.items() if k in df})
 
-        new_frame = self._frame_mgr_cls.lazy_map_partitions(
-            self._partitions, astype_builder
-        )
+        new_frame = self._frame_mgr_cls.map_partitions(self._partitions, astype_builder)
         return self.__constructor__(
             new_frame,
             self.index,
@@ -1145,7 +1143,7 @@ def _map_reduce(self, axis, map_func, reduce_func=None, preserve_index=True):
         else:
             reduce_func = self._build_mapreduce_func(axis, reduce_func)
 
-        map_parts = self._frame_mgr_cls.lazy_map_partitions(self._partitions, map_func)
+        map_parts = self._frame_mgr_cls.map_partitions(self._partitions, map_func)
         reduce_parts = self._frame_mgr_cls.map_axis_partitions(
             axis, map_parts, reduce_func
         )
@@ -1171,7 +1169,7 @@ def _map(self, func, dtypes=None, validate_index=False, validate_columns=False):
         -------
             A new dataframe.
         """
-        new_partitions = self._frame_mgr_cls.lazy_map_partitions(self._partitions, func)
+        new_partitions = self._frame_mgr_cls.map_partitions(self._partitions, func)
         if dtypes == "copy":
             dtypes = self._dtypes
         elif dtypes is not None:
diff --git a/modin/engines/base/frame/partition_manager.py b/modin/engines/base/frame/partition_manager.py
--- a/modin/engines/base/frame/partition_manager.py
+++ b/modin/engines/base/frame/partition_manager.py
@@ -204,7 +204,7 @@ def broadcast_apply(cls, axis, apply_func, left, right, other_name="r"):
         new_partitions = np.array(
             [
                 [
-                    part.add_to_apply_calls(
+                    part.apply(
                         apply_func,
                         **{other_name: right[col_idx] if axis else right[row_idx]},
                     )
@@ -587,9 +587,7 @@ def _apply_func_to_list_of_partitions_broadcast(
     ):
         preprocessed_func = cls.preprocess_func(func)
         return [
-            obj.add_to_apply_calls(
-                preprocessed_func, other=[o.get() for o in broadcasted], **kwargs
-            )
+            obj.apply(preprocessed_func, other=[o.get() for o in broadcasted], **kwargs)
             for obj, broadcasted in zip(partitions, other.T)
         ]
 
@@ -608,9 +606,7 @@ def _apply_func_to_list_of_partitions(cls, func, partitions, **kwargs):
             A list of BaseFramePartition objects.
         """
         preprocessed_func = cls.preprocess_func(func)
-        return [
-            obj.add_to_apply_calls(preprocessed_func, **kwargs) for obj in partitions
-        ]
+        return [obj.apply(preprocessed_func, **kwargs) for obj in partitions]
 
     @classmethod
     def apply_func_to_select_indices(


Test patch: