Repository: modin-project/modin

Issue:
Memory leak when reading CSV files in a loop
### System information
- **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**:

Ubuntu 20.04.3 LTS

- **Modin version** (`modin.__version__`):

0.11.0

- **Python version**:

Python 3.9.7

- **Code we can use to reproduce**:

This is a reproducer made from Modin example
```python
import sys
import time
import modin.pandas as pd
from modin.experimental.engines.omnisci_on_native.frame.omnisci_worker import OmnisciServer


def read(filename):
    columns_names = [
        "trip_id",
        "vendor_id",
        "pickup_datetime",
        "dropoff_datetime",
        "store_and_fwd_flag",
        "rate_code_id",
        "pickup_longitude",
        "pickup_latitude",
        "dropoff_longitude",
        "dropoff_latitude",
        "passenger_count",
        "trip_distance",
        "fare_amount",
        "extra",
        "mta_tax",
        "tip_amount",
        "tolls_amount",
        "ehail_fee",
        "improvement_surcharge",
        "total_amount",
        "payment_type",
        "trip_type",
        "pickup",
        "dropoff",
        "cab_type",
        "precipitation",
        "snow_depth",
        "snowfall",
        "max_temperature",
        "min_temperature",
        "average_wind_speed",
        "pickup_nyct2010_gid",
        "pickup_ctlabel",
        "pickup_borocode",
        "pickup_boroname",
        "pickup_ct2010",
        "pickup_boroct2010",
        "pickup_cdeligibil",
        "pickup_ntacode",
        "pickup_ntaname",
        "pickup_puma",
        "dropoff_nyct2010_gid",
        "dropoff_ctlabel",
        "dropoff_borocode",
        "dropoff_boroname",
        "dropoff_ct2010",
        "dropoff_boroct2010",
        "dropoff_cdeligibil",
        "dropoff_ntacode",
        "dropoff_ntaname",
        "dropoff_puma",
    ]
    # use string instead of category
    columns_types = [
        "int64",
        "string",
        "timestamp",
        "timestamp",
        "string",
        "int64",
        "float64",
        "float64",
        "float64",
        "float64",
        "int64",
        "float64",
        "float64",
        "float64",
        "float64",
        "float64",
        "float64",
        "float64",
        "float64",
        "float64",
        "string",
        "float64",
        "string",
        "string",
        "string",
        "float64",
        "int64",
        "float64",
        "int64",
        "int64",
        "float64",
        "float64",
        "float64",
        "float64",
        "string",
        "float64",
        "float64",
        "string",
        "string",
        "string",
        "float64",
        "float64",
        "float64",
        "float64",
        "string",
        "float64",
        "float64",
        "string",
        "string",
        "string",
        "float64",
    ]

    dtypes = {columns_names[i]: columns_types[i] for i in range(len(columns_names))}
    all_but_dates = {
        col: valtype
        for (col, valtype) in dtypes.items()
        if valtype not in ["timestamp"]
    }
    dates_only = [col for (col, valtype) in dtypes.items() if valtype in ["timestamp"]]

    df = pd.read_csv(
        filename,
        names=columns_names,
        dtype=all_but_dates,
        parse_dates=dates_only,
    )

    df.shape  # to trigger real execution
    df._query_compiler._modin_frame._partitions[0][
        0
    ].frame_id = OmnisciServer().put_arrow_to_omnisci(
        df._query_compiler._modin_frame._partitions[0][0].get()
    )  # to trigger real execution
    return df


def q1_omnisci(df):
    q1_pandas_output = df.groupby("cab_type").size()
    q1_pandas_output.shape  # to trigger real execution
    return q1_pandas_output


def q2_omnisci(df):
    q2_pandas_output = df.groupby("passenger_count").agg({"total_amount": "mean"})
    q2_pandas_output.shape  # to trigger real execution
    return q2_pandas_output


def q3_omnisci(df):
    df["pickup_datetime"] = df["pickup_datetime"].dt.year
    q3_pandas_output = df.groupby(["passenger_count", "pickup_datetime"]).size()
    q3_pandas_output.shape  # to trigger real execution
    return q3_pandas_output


def q4_omnisci(df):
    df["pickup_datetime"] = df["pickup_datetime"].dt.year
    df["trip_distance"] = df["trip_distance"].astype("int64")
    q4_pandas_output = (
        df.groupby(["passenger_count", "pickup_datetime", "trip_distance"], sort=False)
        .size()
        .reset_index()
        .sort_values(
            by=["pickup_datetime", 0], ignore_index=True, ascending=[True, False]
        )
    )
    q4_pandas_output.shape  # to trigger real execution
    return q4_pandas_output


def measure(name, func, *args, **kw):
    t0 = time.time()
    res = func(*args, **kw)
    t1 = time.time()
    print(f"{name}: {t1 - t0} sec")
    return res


def main():
    if len(sys.argv) != 2:
        print(
            f"USAGE: python nyc-taxi-omnisci.py <data file name>"
        )
        return
    for i in range(0, 20):
        print("Iteration #", i+1)
        df = measure("Reading", read, sys.argv[1])
        #measure("Q1", q1_omnisci, df)
        #measure("Q2", q2_omnisci, df)
        #measure("Q3", q3_omnisci, df.copy())
        #measure("Q4", q4_omnisci, df.copy())


if __name__ == "__main__":
    main()
```

<!--
You can obtain the Modin version with

python -c "import modin; print(modin.__version__)"
-->

### Describe the problem
<!-- Describe the problem clearly here. -->

Download data files form https://modin-datasets.s3.amazonaws.com/taxi/trips_xaa.csv

After CSV files are loaded and no longer needed (python should garbage collect a dataframe) native resources occupied by CSV data are not released. This leads to a memory leak. On system with tight memory condition even 2nd iteration of a benchmark may fail because resources from 1st iteration are still kept in memory.

### Source code / logs
<!-- Include any logs or source code that would be helpful to diagnose the problem. If including tracebacks, please include the full traceback. Large logs and files should be attached. Try to provide a reproducible test case that is the bare minimum necessary to generate the problem. -->



Discussion:
cc @ienkovich 
The source of the problem is that we don't drop tables when a corresponding partition dies. We need to add a proper finalizer to fix that.
I thought that it comes from loading CSV data using FSI, but you probably know better.
We don't actually use FSI to parse CSV. We parse it using `pyarrow` into arrow Table. Arrow table is stored in a partition and should be freed upon its death. But if we import the data into OmniSci as a table then it's stuck there forever. 
Do you mean that there is no general case of garbage collecting data frames imported in omnisci? Do all frames that get imported in omnisci remain in memory or are you talking about disk storage?
From OmniSci point of view Modin simply executes `CREATE TEMPORARY TABLE ...` DDL query to import frame partition (we do it through special method `importArrowTable`, but the result is the same). Temporary tables are in-memory only and they are stored in memory until `DROP TABLE ...` is executed (or until process is finished). Frame partitions can be garbage collected in Python but this doesn't trigger `DROP TABLE` query execution. Suppose, we could add `__del__` method to `OmnisciOnNativeFramePartition` class. Something like that:
```
def __del__(self):
  if self.frame_id is not None:
    omniSession = OmnisciServer()
    omniSession.executeDDL(f"DROP TABLE {self.frame_id};")
```
But here I'm not sure if it is actually safe to assume that OmniSci server still exists when partition is destroyed. Can this method be called on a program shutdown after OmniSci is already destroyed?
We could register a function, which will be executed at program termination, with `atexit` but is it okay to appeal to OmniSci when it has already been destroyed?
The original issue is in keepin data in OmniSci for data frames that were garbage collected. Having a cleanup at program termination only wouldn't help to resolve the issue. We should be able to have a loop where we read a CSV file and perform some computations on it. With proper resource management, we can do any number of such iterations but right now we would run out of memory at some point due to the constantly increasing number of in-memory tables stored in OmniSci.
Your suggestion related to `__del__` method will probably help to clean up in-memory tables stored in OmniSci if we are in a for loop.
I think @ienkovich is worried that `__del__` may be called on application shutdown and we may attempt to contact omnisci server when/after it is shut down which may cause intermittent crash/raise at the end of application.
> I think @ienkovich is worried that `__del__` may be called on application shutdown and we may attempt to contact omnisci server when/after it is shut down which may cause intermittent crash/raise at the end of application.

That's right. Basically, such `__del__` method looks good for me, it would allow to free the memory hold by OmniSci for garbage collected DataFrames, but I'm not experienced enough in Python's way to destroy objects and I'm worried about the case when we destroy a partition object after OmniSci server shutdown. Can you advise something here? Can we (should we) have some check for already destroyed OmniSci server in this finilizer?
As I see, currently we create new instance of the `OmnisciServer` every time when we want to send a query to it (since we do not have `._server` member initialized at the class level, only at the instance level). So we should definitely check for destroyed OmniSci server in the finalizer and create new instance to send the query. Also, I would like to notice the approach with explicit `PyDbEngine` initialization mentioned in #3191 is not possible for now because of the instance level initialization as I said above.
> As I see, currently we create new instance of the `OmnisciServer` every time when we want to send a query to it (since we do not have `._server` member initialized at the class level, only at the instance level).

All methods of `OmnisciServer` are `classmethod`. Instances are created only to trigger the initialization on the first creation.
True, you are right. I didn't pay attention to the decorator. Anyway, a check for destroyed OmniSci server in the finalizer should be presented and if it is, we have to create a new instance to send the query.
>Anyway, a check for destroyed OmniSci server in the finalizer should be presented and if it is, we have to create a new instance to send the query.

All data is imported as temporary tables and dies with the server, so no reason to create a new instance.
In that case since we don't know order of garbage collection while reclaiming the memory (what will be removed first: `OmnisciServer` or `df._query_compiler._modin_frame`) we could add `OmnisciServer` as a member to `df._query_compiler._modin_frame` and implement `__del__` for `_modin_frame` in which there will be stuff related to clean up of in-memory tables.
I tried to put a destructor to drop temporary tables on partitions removal, but it doesn't seem to help. Looks like `DROP TABLE` execution doesn't really free the memory in OmniSci.

Resolving patch:
diff --git a/modin/experimental/core/execution/native/implementations/omnisci_on_native/partitioning/partition.py b/modin/experimental/core/execution/native/implementations/omnisci_on_native/partitioning/partition.py
--- a/modin/experimental/core/execution/native/implementations/omnisci_on_native/partitioning/partition.py
+++ b/modin/experimental/core/execution/native/implementations/omnisci_on_native/partitioning/partition.py
@@ -16,6 +16,9 @@
 import pandas
 
 from modin.core.dataframe.pandas.partitioning.partition import PandasDataframePartition
+from ..omnisci_worker import (
+    OmnisciServer,
+)
 import pyarrow
 
 
@@ -62,6 +65,12 @@ def __init__(
         self.arrow_table = arrow_table
         self._length_cache = length
         self._width_cache = width
+        self._server = OmnisciServer
+
+    def __del__(self):
+        """Deallocate OmniSci resources related to the partition."""
+        if self.frame_id is not None:
+            self._server.executeDDL(f"DROP TABLE {self.frame_id};")
 
     def to_pandas(self):
         """


Test patch: