Outliers filter and cleaning

app.py

@@ -160,6 +160,12 @@ def prepare_data():
     )
     invalid_trades["creation_date"] = invalid_trades["creation_timestamp"].dt.date
 
+    # discovering outliers for ROI
+    outliers = trades_df.loc[trades_df["roi"] >= 1000]
+    if len(outliers) > 0:
+        outliers.to_parquet("./tmp/outliers.parquet")
+        trades_df = trades_df.loc[trades_df["roi"] < 1000]
+
     return tools_df, trades_df, tools_accuracy_info, invalid_trades
 
 

tabs/metrics.py

@@ -24,49 +24,6 @@ HEIGHT = 600
 WIDTH = 1000
 
 
-def plot_trade_details(metric_name: str, trades_df: pd.DataFrame) -> gr.LinePlot:
-    """Plots the trade details for the given trade detail."""
-    column_name = metric_name
-    if metric_name == "mech calls":
-        metric_name = "mech_calls"
-        column_name = "num_mech_calls"
-    elif metric_name == "ROI":
-        column_name = "roi"
-
-    # this is to filter out the data before 2023-09-01
-    trades_filtered = trades_df[trades_df["creation_timestamp"] > "2023-09-01"]
-    trades_filtered = (
-        trades_filtered.groupby("month_year_week")[column_name]
-        .quantile([0.25, 0.5, 0.75])
-        .unstack()
-    )
-    trades_filtered.columns = trades_filtered.columns.astype(str)
-    trades_filtered.reset_index(inplace=True)
-    trades_filtered.columns = [
-        "month_year_week",
-        "25th_percentile",
-        "50th_percentile",
-        "75th_percentile",
-    ]
-    # reformat the data as percentile, date, value
-    trades_filtered = trades_filtered.melt(
-        id_vars=["month_year_week"], var_name="percentile", value_name=metric_name
-    )
-
-    return gr.LinePlot(
-        value=trades_filtered,
-        x="month_year_week",
-        y=metric_name,
-        color="percentile",
-        show_label=True,
-        interactive=True,
-        show_actions_button=True,
-        tooltip=["month_year_week", "percentile", metric_name],
-        height=HEIGHT,
-        width=WIDTH,
-    )
-
-
 def get_metrics(
     metric_name: str, column_name: str, market_creator: str, trades_df: pd.DataFrame
 ) -> pd.DataFrame:
@@ -107,7 +64,6 @@ def get_boxplot_metrics(column_name: str, trades_df: pd.DataFrame) -> pd.DataFra
     trades_filtered = trades_filtered[
         ["creation_timestamp", "month_year_week", "market_creator", column_name]
     ]
-
     # adding the total
     trades_filtered_all = trades_filtered.copy(deep=True)
     trades_filtered_all["market_creator"] = "all"