Repository: modin-project/modin

Issue:
Convert docker container examples to use external data files instead of packing them inside
**Is your feature request related to a problem? Please describe.**
A clear and concise description of what the problem is. What kind of performance improvements would you like to see with this new API?

Currently we build example docker containers by packing datasets inside of container image. It has two disadvantages (1) huge image size (2) it is inflexible because doesn't allow to run example on a different dataset. It is possible to map a filesystem directory inside a container with `-v` option to `docker run` command. This way we can let user to run example container image on any dataset that he likes.


Discussion:

Resolving patch:
diff --git a/examples/docker/census-on-omnisci/census-omnisci.py b/examples/docker/census-on-omnisci/census-omnisci.py
--- a/examples/docker/census-on-omnisci/census-omnisci.py
+++ b/examples/docker/census-on-omnisci/census-omnisci.py
@@ -12,6 +12,7 @@
 # governing permissions and limitations under the License.
 
 import os
+import sys
 import time
 import modin.pandas as pd
 from modin.experimental.engines.omnisci_on_ray.frame.omnisci_worker import OmnisciServer
@@ -25,25 +26,105 @@
 import numpy as np
 
 
-def read():
+def read(filename):
     columns_names = [
-        "YEAR0", "DATANUM", "SERIAL", "CBSERIAL", "HHWT", "CPI99", "GQ", "QGQ", "PERNUM", "PERWT", "SEX",
-        "AGE", "EDUC", "EDUCD", "INCTOT", "SEX_HEAD", "SEX_MOM", "SEX_POP", "SEX_SP", "SEX_MOM2", "SEX_POP2",
-        "AGE_HEAD", "AGE_MOM", "AGE_POP", "AGE_SP", "AGE_MOM2", "AGE_POP2", "EDUC_HEAD", "EDUC_MOM", "EDUC_POP",
-        "EDUC_SP", "EDUC_MOM2", "EDUC_POP2", "EDUCD_HEAD", "EDUCD_MOM", "EDUCD_POP", "EDUCD_SP", "EDUCD_MOM2",
-        "EDUCD_POP2", "INCTOT_HEAD", "INCTOT_MOM", "INCTOT_POP", "INCTOT_SP", "INCTOT_MOM2", "INCTOT_POP2",
+        "YEAR0",
+        "DATANUM",
+        "SERIAL",
+        "CBSERIAL",
+        "HHWT",
+        "CPI99",
+        "GQ",
+        "QGQ",
+        "PERNUM",
+        "PERWT",
+        "SEX",
+        "AGE",
+        "EDUC",
+        "EDUCD",
+        "INCTOT",
+        "SEX_HEAD",
+        "SEX_MOM",
+        "SEX_POP",
+        "SEX_SP",
+        "SEX_MOM2",
+        "SEX_POP2",
+        "AGE_HEAD",
+        "AGE_MOM",
+        "AGE_POP",
+        "AGE_SP",
+        "AGE_MOM2",
+        "AGE_POP2",
+        "EDUC_HEAD",
+        "EDUC_MOM",
+        "EDUC_POP",
+        "EDUC_SP",
+        "EDUC_MOM2",
+        "EDUC_POP2",
+        "EDUCD_HEAD",
+        "EDUCD_MOM",
+        "EDUCD_POP",
+        "EDUCD_SP",
+        "EDUCD_MOM2",
+        "EDUCD_POP2",
+        "INCTOT_HEAD",
+        "INCTOT_MOM",
+        "INCTOT_POP",
+        "INCTOT_SP",
+        "INCTOT_MOM2",
+        "INCTOT_POP2",
     ]
     columns_types = [
-        "int64", "int64", "int64", "float64", "int64", "float64", "int64", "float64", "int64", "int64",
-        "int64", "int64", "int64", "int64", "int64", "float64", "float64", "float64", "float64", "float64",
-        "float64", "float64", "float64", "float64", "float64", "float64", "float64", "float64", "float64",
-        "float64", "float64", "float64", "float64", "float64", "float64", "float64", "float64", "float64",
-        "float64", "float64", "float64", "float64", "float64", "float64", "float64",
+        "int64",
+        "int64",
+        "int64",
+        "float64",
+        "int64",
+        "float64",
+        "int64",
+        "float64",
+        "int64",
+        "int64",
+        "int64",
+        "int64",
+        "int64",
+        "int64",
+        "int64",
+        "float64",
+        "float64",
+        "float64",
+        "float64",
+        "float64",
+        "float64",
+        "float64",
+        "float64",
+        "float64",
+        "float64",
+        "float64",
+        "float64",
+        "float64",
+        "float64",
+        "float64",
+        "float64",
+        "float64",
+        "float64",
+        "float64",
+        "float64",
+        "float64",
+        "float64",
+        "float64",
+        "float64",
+        "float64",
+        "float64",
+        "float64",
+        "float64",
+        "float64",
+        "float64",
     ]
     dtypes = {columns_names[i]: columns_types[i] for i in range(len(columns_names))}
 
     df = pd.read_csv(
-        os.path.expanduser('~/ipums_education2income_1970-2010.csv'),
+        filename,
         names=columns_names,
         dtype=dtypes,
         skiprows=1,
@@ -60,9 +141,30 @@ def read():
 
 def etl(df):
     keep_cols = [
-        "YEAR0", "DATANUM", "SERIAL", "CBSERIAL", "HHWT", "CPI99", "GQ", "PERNUM", "SEX", "AGE",
-        "INCTOT", "EDUC", "EDUCD", "EDUC_HEAD", "EDUC_POP", "EDUC_MOM", "EDUCD_MOM2", "EDUCD_POP2",
-        "INCTOT_MOM", "INCTOT_POP", "INCTOT_MOM2", "INCTOT_POP2", "INCTOT_HEAD", "SEX_HEAD",
+        "YEAR0",
+        "DATANUM",
+        "SERIAL",
+        "CBSERIAL",
+        "HHWT",
+        "CPI99",
+        "GQ",
+        "PERNUM",
+        "SEX",
+        "AGE",
+        "INCTOT",
+        "EDUC",
+        "EDUCD",
+        "EDUC_HEAD",
+        "EDUC_POP",
+        "EDUC_MOM",
+        "EDUCD_MOM2",
+        "EDUCD_POP2",
+        "INCTOT_MOM",
+        "INCTOT_POP",
+        "INCTOT_MOM2",
+        "INCTOT_POP2",
+        "INCTOT_HEAD",
+        "SEX_HEAD",
     ]
     df = df[keep_cols]
 
@@ -143,20 +245,27 @@ def measure(name, func, *args, **kw):
     t0 = time.time()
     res = func(*args, **kw)
     t1 = time.time()
-    print(f'{name}: {t1 - t0} sec')
+    print(f"{name}: {t1 - t0} sec")
     return res
 
 
 def main():
+    if len(sys.argv) != 2:
+        print(
+            f"USAGE: docker run --rm -v /path/to/dataset:/dataset census-omnisci <data file name in /path/to/dataset>"
+        )
+        return
     # ML specific
     N_RUNS = 50
     TEST_SIZE = 0.1
     RANDOM_STATE = 777
 
-    df = measure('Reading', read)
-    _, X, y = measure('ETL', etl, df)
-    measure('ML', ml, X, y, random_state=RANDOM_STATE, n_runs=N_RUNS, test_size=TEST_SIZE)
+    df = measure("Reading", read, os.path.join("/dataset", sys.argv[1]))
+    _, X, y = measure("ETL", etl, df)
+    measure(
+        "ML", ml, X, y, random_state=RANDOM_STATE, n_runs=N_RUNS, test_size=TEST_SIZE
+    )
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()
diff --git a/examples/docker/plasticc-on-omnisci/plasticc-omnisci.py b/examples/docker/plasticc-on-omnisci/plasticc-omnisci.py
--- a/examples/docker/plasticc-on-omnisci/plasticc-omnisci.py
+++ b/examples/docker/plasticc-on-omnisci/plasticc-omnisci.py
@@ -12,6 +12,7 @@
 # governing permissions and limitations under the License.
 
 import os
+import sys
 import time
 from collections import OrderedDict
 from functools import partial
@@ -43,8 +44,18 @@ def create_dtypes():
 
     # load metadata
     columns_names = [
-        "object_id", "ra", "decl", "gal_l", "gal_b", "ddf", "hostgal_specz",
-        "hostgal_photoz", "hostgal_photoz_err", "distmod", "mwebv", "target",
+        "object_id",
+        "ra",
+        "decl",
+        "gal_l",
+        "gal_b",
+        "ddf",
+        "hostgal_specz",
+        "hostgal_photoz",
+        "hostgal_photoz_err",
+        "distmod",
+        "mwebv",
+        "target",
     ]
     meta_dtypes = ["int32"] + ["float32"] * 4 + ["int32"] + ["float32"] * 5 + ["int32"]
     meta_dtypes = OrderedDict(
@@ -69,7 +80,7 @@ def measure(name, func, *args, **kw):
     t0 = time.time()
     res = func(*args, **kw)
     t1 = time.time()
-    print(f'{name}: {t1 - t0} sec')
+    print(f"{name}: {t1 - t0} sec")
     return res
 
 
@@ -120,24 +131,34 @@ def multi_weighted_logloss(y_true, y_preds, classes, class_weights):
 
 
 def xgb_multi_weighted_logloss(y_predicted, y_true, classes, class_weights):
-    loss = multi_weighted_logloss(y_true.get_label(), y_predicted, classes, class_weights)
+    loss = multi_weighted_logloss(
+        y_true.get_label(), y_predicted, classes, class_weights
+    )
     return "wloss", loss
 
+
 ################ helper functions ###############################
 
 
-def read(dtypes, meta_dtypes):
-    train = pd.read_csv(os.path.expanduser("~/training_set.csv"), dtype=dtypes)
+def read(
+    training_set_filename,
+    test_set_filename,
+    training_set_metadata_filename,
+    test_set_metadata_filename,
+    dtypes,
+    meta_dtypes,
+):
+    train = pd.read_csv(training_set_filename, dtype=dtypes)
     test = pd.read_csv(
-        os.path.expanduser("~/test_set_skiprows.csv"),
+        test_set_filename,
         names=list(dtypes.keys()),
         dtype=dtypes,
         header=0,
     )
 
-    train_meta = pd.read_csv(os.path.expanduser("~/training_set_metadata.csv"), dtype=meta_dtypes)
+    train_meta = pd.read_csv(training_set_metadata_filename, dtype=meta_dtypes)
     target = meta_dtypes.pop("target")
-    test_meta = pd.read_csv(os.path.expanduser("~/test_set_metadata.csv"), dtype=meta_dtypes)
+    test_meta = pd.read_csv(test_set_metadata_filename, dtype=meta_dtypes)
     meta_dtypes["target"] = target
 
     dfs = (train, train_meta, test, test_meta)
@@ -167,7 +188,9 @@ def etl(df, df_meta):
 
     agg_df["flux_diff"] = agg_df["flux_max"] - agg_df["flux_min"]
     agg_df["flux_dif2"] = agg_df["flux_diff"] / agg_df["flux_mean"]
-    agg_df["flux_w_mean"] = agg_df["flux_by_flux_ratio_sq_sum"] / agg_df["flux_ratio_sq_sum"]
+    agg_df["flux_w_mean"] = (
+        agg_df["flux_by_flux_ratio_sq_sum"] / agg_df["flux_ratio_sq_sum"]
+    )
     agg_df["flux_dif3"] = agg_df["flux_diff"] / agg_df["flux_w_mean"]
     agg_df["mjd_diff"] = agg_df["mjd_max"] - agg_df["mjd_min"]
 
@@ -184,7 +207,9 @@ def etl(df, df_meta):
 
 
 def ml(train_final, test_final):
-    X_train, y_train, X_test, y_test, Xt, classes, class_weights = split_step(train_final, test_final)
+    X_train, y_train, X_test, y_test, Xt, classes, class_weights = split_step(
+        train_final, test_final
+    )
 
     cpu_params = {
         "objective": "multi:softprob",
@@ -198,7 +223,9 @@ def ml(train_final, test_final):
         "colsample_bytree": 0.7,
     }
 
-    func_loss = partial(xgb_multi_weighted_logloss, classes=classes, class_weights=class_weights)
+    func_loss = partial(
+        xgb_multi_weighted_logloss, classes=classes, class_weights=class_weights
+    )
 
     dtrain = xgb.DMatrix(data=X_train, label=y_train)
     dvalid = xgb.DMatrix(data=X_test, label=y_test)
@@ -224,14 +251,31 @@ def ml(train_final, test_final):
 
 
 def main():
+    if len(sys.argv) != 5:
+        print(
+            f"USAGE: docker run --rm -v /path/to/dataset:/dataset plasticc-omnisci <training set file name in /path/to/dataset> <test set file name in /path/to/dataset> <training set metadata file name in /path/to/dataset> <test set metadata file name in /path/to/dataset>"
+        )
+        return
+
     dtypes, meta_dtypes = create_dtypes()
 
-    train, train_meta, test, test_meta = measure('Reading', read, dtypes, meta_dtypes)
-    train_final, test_final = measure("ETL", all_etl, train, train_meta, test, test_meta)
+    train, train_meta, test, test_meta = measure(
+        "Reading",
+        read,
+        os.path.join("/dataset", sys.argv[1]),
+        os.path.join("/dataset", sys.argv[2]),
+        os.path.join("/dataset", sys.argv[3]),
+        os.path.join("/dataset", sys.argv[4]),
+        dtypes,
+        meta_dtypes,
+    )
+    train_final, test_final = measure(
+        "ETL", all_etl, train, train_meta, test, test_meta
+    )
     cpu_loss = measure("ML", ml, train_final, test_final)
 
     print("validation cpu_loss:", cpu_loss)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()
diff --git a/examples/docker/taxi-on-omnisci/nyc-taxi-omnisci.py b/examples/docker/taxi-on-omnisci/nyc-taxi-omnisci.py
--- a/examples/docker/taxi-on-omnisci/nyc-taxi-omnisci.py
+++ b/examples/docker/taxi-on-omnisci/nyc-taxi-omnisci.py
@@ -12,42 +12,131 @@
 # governing permissions and limitations under the License.
 
 import os
+import sys
 import time
 import modin.pandas as pd
 from modin.experimental.engines.omnisci_on_ray.frame.omnisci_worker import OmnisciServer
 
-def read():
+
+def read(filename):
     columns_names = [
-        "trip_id", "vendor_id", "pickup_datetime", "dropoff_datetime", "store_and_fwd_flag",
-        "rate_code_id", "pickup_longitude", "pickup_latitude", "dropoff_longitude", "dropoff_latitude",
-        "passenger_count", "trip_distance", "fare_amount", "extra", "mta_tax", "tip_amount",
-        "tolls_amount", "ehail_fee", "improvement_surcharge", "total_amount", "payment_type",
-        "trip_type", "pickup", "dropoff", "cab_type", "precipitation", "snow_depth", "snowfall",
-        "max_temperature", "min_temperature", "average_wind_speed", "pickup_nyct2010_gid",
-        "pickup_ctlabel", "pickup_borocode", "pickup_boroname", "pickup_ct2010",
-        "pickup_boroct2010", "pickup_cdeligibil", "pickup_ntacode", "pickup_ntaname", "pickup_puma",
-        "dropoff_nyct2010_gid", "dropoff_ctlabel", "dropoff_borocode", "dropoff_boroname",
-        "dropoff_ct2010", "dropoff_boroct2010", "dropoff_cdeligibil", "dropoff_ntacode",
-        "dropoff_ntaname", "dropoff_puma",
+        "trip_id",
+        "vendor_id",
+        "pickup_datetime",
+        "dropoff_datetime",
+        "store_and_fwd_flag",
+        "rate_code_id",
+        "pickup_longitude",
+        "pickup_latitude",
+        "dropoff_longitude",
+        "dropoff_latitude",
+        "passenger_count",
+        "trip_distance",
+        "fare_amount",
+        "extra",
+        "mta_tax",
+        "tip_amount",
+        "tolls_amount",
+        "ehail_fee",
+        "improvement_surcharge",
+        "total_amount",
+        "payment_type",
+        "trip_type",
+        "pickup",
+        "dropoff",
+        "cab_type",
+        "precipitation",
+        "snow_depth",
+        "snowfall",
+        "max_temperature",
+        "min_temperature",
+        "average_wind_speed",
+        "pickup_nyct2010_gid",
+        "pickup_ctlabel",
+        "pickup_borocode",
+        "pickup_boroname",
+        "pickup_ct2010",
+        "pickup_boroct2010",
+        "pickup_cdeligibil",
+        "pickup_ntacode",
+        "pickup_ntaname",
+        "pickup_puma",
+        "dropoff_nyct2010_gid",
+        "dropoff_ctlabel",
+        "dropoff_borocode",
+        "dropoff_boroname",
+        "dropoff_ct2010",
+        "dropoff_boroct2010",
+        "dropoff_cdeligibil",
+        "dropoff_ntacode",
+        "dropoff_ntaname",
+        "dropoff_puma",
     ]
     # use string instead of category
     columns_types = [
-        "int64", "string", "timestamp", "timestamp", "string", "int64", "float64", "float64",
-        "float64", "float64", "int64", "float64", "float64", "float64", "float64", "float64", "float64",
-        "float64", "float64", "float64", "string", "float64", "string", "string", "string", "float64",
-        "int64", "float64", "int64", "int64", "float64", "float64", "float64", "float64", "string", "float64",
-        "float64", "string", "string", "string", "float64", "float64", "float64", "float64", "string",
-        "float64", "float64", "string", "string", "string", "float64",
+        "int64",
+        "string",
+        "timestamp",
+        "timestamp",
+        "string",
+        "int64",
+        "float64",
+        "float64",
+        "float64",
+        "float64",
+        "int64",
+        "float64",
+        "float64",
+        "float64",
+        "float64",
+        "float64",
+        "float64",
+        "float64",
+        "float64",
+        "float64",
+        "string",
+        "float64",
+        "string",
+        "string",
+        "string",
+        "float64",
+        "int64",
+        "float64",
+        "int64",
+        "int64",
+        "float64",
+        "float64",
+        "float64",
+        "float64",
+        "string",
+        "float64",
+        "float64",
+        "string",
+        "string",
+        "string",
+        "float64",
+        "float64",
+        "float64",
+        "float64",
+        "string",
+        "float64",
+        "float64",
+        "string",
+        "string",
+        "string",
+        "float64",
     ]
 
     dtypes = {columns_names[i]: columns_types[i] for i in range(len(columns_names))}
     all_but_dates = {
-        col: valtype for (col, valtype) in dtypes.items() if valtype not in ["timestamp"]
+        col: valtype
+        for (col, valtype) in dtypes.items()
+        if valtype not in ["timestamp"]
     }
     dates_only = [col for (col, valtype) in dtypes.items() if valtype in ["timestamp"]]
 
     df = pd.read_csv(
-        os.path.expanduser('~/trips_xaa.csv'),
+        filename,
         names=columns_names,
         dtype=all_but_dates,
         parse_dates=dates_only,
@@ -67,17 +156,20 @@ def q1_omnisci(df):
     q1_pandas_output.shape  # to trigger real execution
     return q1_pandas_output
 
+
 def q2_omnisci(df):
     q2_pandas_output = df.groupby("passenger_count").agg({"total_amount": "mean"})
     q2_pandas_output.shape  # to trigger real execution
     return q2_pandas_output
 
+
 def q3_omnisci(df):
     df["pickup_datetime"] = df["pickup_datetime"].dt.year
     q3_pandas_output = df.groupby(["passenger_count", "pickup_datetime"]).size()
     q3_pandas_output.shape  # to trigger real execution
     return q3_pandas_output
 
+
 def q4_omnisci(df):
     df["pickup_datetime"] = df["pickup_datetime"].dt.year
     df["trip_distance"] = df["trip_distance"].astype("int64")
@@ -85,24 +177,34 @@ def q4_omnisci(df):
         df.groupby(["passenger_count", "pickup_datetime", "trip_distance"], sort=False)
         .size()
         .reset_index()
-        .sort_values(by=["pickup_datetime", 0], ignore_index=True, ascending=[True, False])
+        .sort_values(
+            by=["pickup_datetime", 0], ignore_index=True, ascending=[True, False]
+        )
     )
     q4_pandas_output.shape  # to trigger real execution
     return q4_pandas_output
 
+
 def measure(name, func, *args, **kw):
     t0 = time.time()
     res = func(*args, **kw)
     t1 = time.time()
-    print(f'{name}: {t1 - t0} sec')
+    print(f"{name}: {t1 - t0} sec")
     return res
 
+
 def main():
-    df = measure('Reading', read)
-    measure('Q1', q1_omnisci, df)
-    measure('Q2', q2_omnisci, df)
-    measure('Q3', q3_omnisci, df.copy())
-    measure('Q4', q4_omnisci, df.copy())
+    if len(sys.argv) != 2:
+        print(
+            f"USAGE: docker run --rm -v /path/to/dataset:/dataset nyc-taxi-omnisci <data file name in /path/to/dataset>"
+        )
+        return
+    df = measure("Reading", read, os.path.join("/dataset", sys.argv[1]))
+    measure("Q1", q1_omnisci, df)
+    measure("Q2", q2_omnisci, df)
+    measure("Q3", q3_omnisci, df.copy())
+    measure("Q4", q4_omnisci, df.copy())
+
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()


Test patch: