Repository: yasstake/rbot (language: python)

Issue:
オーダーリストをpolarsのDataFrameで返すことを検討する。



Discussion / hints:

Fix patch:
diff --git a/src/db/df.rs b/src/db/df.rs
index 3f861f36..ec74753d 100644
--- a/src/db/df.rs
+++ b/src/db/df.rs
@@ -1,3 +1,5 @@
+// Copyright(c) 2022-2023. yasstake. All rights reserved.
+
 use crate::common::Trade;
 use crate::common::{time_string, MicroSec, SEC};
 use polars::prelude::BooleanType;
@@ -12,7 +14,7 @@ use polars_core::prelude::SortOptions;
 use polars_lazy::prelude::IntoLazy;
 use polars_lazy::prelude::{col, LazyFrame};
 use polars_time::ClosedWindow;
-// Copyright(c) 2022. yasstake. All rights reserved.
+
 
 #[allow(non_upper_case_globals)]
 #[allow(non_snake_case)]
@@ -485,13 +487,21 @@ impl AsDynamicGroupOptions for DynamicGroupOptions {
     }
 }
 
+pub fn convert_timems_to_datetime(df: &mut DataFrame) -> &DataFrame {
+    let time = df.column(KEY::time_stamp).unwrap().i64().unwrap().clone();
+    let date_time = time.into_datetime(TimeUnit::Microseconds, None);
+    let df = df.with_column(date_time).unwrap();
+
+    df
+}
+
 use polars::prelude::*;
 use rust_decimal::prelude::ToPrimitive;
 
 #[cfg(test)]
 mod test_df {
     use super::*;
-    use crate::common::DAYS;
+    use crate::{common::DAYS, db::df};
 
     #[test]
     fn test_simple_dynamic_group() {
@@ -630,4 +640,45 @@ mod test_df {
         let r3 = ohlcv_from_ohlcvv_df(&r2, 0, 0, 10);
         println!("{:?}", r3);
     }
+
+
+    #[test]
+    fn test_make_ohlcv_datetime() {
+        use polars::datatypes::TimeUnit;
+        use polars::datatypes::TimeZone;
+
+        let mut time = Series::new(KEY::time_stamp, Vec::<MicroSec>::new());
+
+        let mut t64 = time.i64().unwrap().clone();
+        let date_time = t64.into_datetime(TimeUnit::Microseconds, None);
+
+        let t = Series::new("time", date_time);
+        let open = Series::new(KEY::open, Vec::<f64>::new());
+        let high = Series::new(KEY::high, Vec::<f64>::new());
+        let low = Series::new(KEY::low, Vec::<f64>::new());
+        let close = Series::new(KEY::close, Vec::<f64>::new());
+        let vol = Series::new(KEY::vol, Vec::<f64>::new());
+        let count = Series::new(KEY::count, Vec::<f64>::new());
+
+
+
+        let df = DataFrame::new(vec![t, open, high, low, close, vol, count]).unwrap();
+    
+        println!("{:?}", df);
+    }
+
+    #[test]
+    fn test_convert_ohlc_datetime() {
+        let mut df = make_empty_ohlcv();
+
+        println!("{:?}", df);
+
+
+        let time = df.column("time_stamp").unwrap().i64().unwrap().clone();
+        let mut date_time = time.into_datetime(TimeUnit::Microseconds, None);
+        let df = df.with_column(date_time).unwrap();
+
+        println!("{:?}", df);
+    }
+
 }
diff --git a/src/db/sqlite.rs b/src/db/sqlite.rs
index 4dce862a..253f5180 100644
--- a/src/db/sqlite.rs
+++ b/src/db/sqlite.rs
@@ -30,6 +30,8 @@ use polars::prelude::Float64Type;
 use std::thread;
 use crossbeam_channel::Sender;
 
+use super::df::convert_timems_to_datetime;
+
 pub trait TradeTableQuery {
     fn open(name: &str) -> Result<Self, Error>
     where
@@ -611,8 +613,10 @@ impl TradeTable {
     ) -> PyResult<PyDataFrame> {
         from_time = TradeTable::ohlcv_start(from_time); // 開始tickは確定足、終了は未確定足もOK.
 
-        let df = self.ohlcvv_df(from_time, to_time, window_sec);
+        let mut df = self.ohlcvv_df(from_time, to_time, window_sec);
         
+        let df = convert_timems_to_datetime(&mut df).clone();
+
         return Ok(PyDataFrame(df));
     }
 
@@ -685,7 +689,8 @@ impl TradeTable {
     ) -> PyResult<PyDataFrame> {
         from_time = TradeTable::ohlcv_start(from_time); // 開始tickは確定足、終了は未確定足もOK.
 
-        let df = self.ohlcv_df(from_time, to_time, window_sec);
+        let mut df = self.ohlcv_df(from_time, to_time, window_sec);
+        let df = convert_timems_to_datetime(&mut df).clone();        
         let df = PyDataFrame(df);
 
         return Ok(df);
@@ -697,9 +702,9 @@ impl TradeTable {
         from_time: MicroSec,
         to_time: MicroSec,
     ) -> PyResult<PyDataFrame> {
-        let trades = self.select_df_from_db(from_time, to_time);
-
-        let df = PyDataFrame(trades);
+        let mut df  = self.select_df_from_db(from_time, to_time);
+        let df = convert_timems_to_datetime(&mut df).clone();        
+        let df = PyDataFrame(df);
 
         return Ok(df);
     }


Test patch:
diff --git a/src/test/binance_test/market.ipynb b/src/test/binance_test/market.ipynb
index 4210b56f..563f0cdc 100644
--- a/src/test/binance_test/market.ipynb
+++ b/src/test/binance_test/market.ipynb
@@ -8,7 +8,7 @@
     {
      "data": {
       "text/html": [
-       "<table border=\"1\"><tr><th>exchange_name</th><td>BN</td></tr><tr><th>trade_category</th><td>TESTSPOT</td></tr><tr><th>trade_symbol</th><td>BTCUSDT</td></tr><tr><th>home_currency</th><td>USDT</td></tr><tr><th>foreign_currency</th><td>BTC</td></tr><tr><th>testnet</th><td>True</td></tr><tr><th>rest_endpoint</th><td>https://testnet.binance.vision</td></tr><tr><th>public_ws_endpoint</th><td>wss://testnet.binance.vision/ws</td></tr><tr><th>private_ws_endpoint</th><td>wss://testnet.binance.vision/ws</td></tr><tr><th>history_web_base</th><td>https://data.binance.vision/data/spot/daily/trades</td></tr><tr><th>new_order_path</th><td>/api/v3/order</td></tr><tr><th>cancel_order_path</th><td>/api/v3/order</td></tr><tr><th>public_subscribe_message</th><td>{&quot;id&quot;:1,&quot;method&quot;:&quot;SUBSCRIBE&quot;,&quot;params&quot;:[&quot;btcusdt@trade&quot;,&quot;btcusdt@depth@100ms&quot;]}</td></tr><tr><th>api_key</th><td>NOhZ*******************</td></tr><tr><th>api_secret</th><td>yPrx*******************</td></tr><tr><th>exchange_config</th><td><table border=\"1\"><tr><th>price_unit</th><td>0.01</td></tr><tr><th>size_unit</th><td>0.00001</td></tr><tr><th>maker_fee</th><td>0.0</td></tr><tr><th>taker_fee</th><td>0.0</td></tr><tr><th>price_type</th><td>Foreign</td></tr><tr><th>fee_type</th><td>Home</td></tr><tr><th>home_currency</th><td>USDT</td></tr><tr><th>foreign_currency</th><td>BTC</td></tr></table></td></tr></table>"
+       "<table border=\"1\"><tr><th>exchange_name</th><td>BN</td></tr><tr><th>trade_category</th><td>TESTSPOT</td></tr><tr><th>trade_symbol</th><td>BTCUSDT</td></tr><tr><th>home_currency</th><td>USDT</td></tr><tr><th>foreign_currency</th><td>BTC</td></tr><tr><th>testnet</th><td>True</td></tr><tr><th>rest_endpoint</th><td>https://testnet.binance.vision</td></tr><tr><th>public_ws_endpoint</th><td>wss://testnet.binance.vision/ws</td></tr><tr><th>private_ws_endpoint</th><td>wss://testnet.binance.vision/ws</td></tr><tr><th>history_web_base</th><td>https://data.binance.vision/data/spot/daily/trades</td></tr><tr><th>new_order_path</th><td>/api/v3/order</td></tr><tr><th>cancel_order_path</th><td>/api/v3/order</td></tr><tr><th>open_orders_path</th><td>/api/v3/openOrders</td></tr><tr><th>account_path</th><td>/api/v3/account</td></tr><tr><th>public_subscribe_message</th><td>{&quot;id&quot;:1,&quot;method&quot;:&quot;SUBSCRIBE&quot;,&quot;params&quot;:[&quot;btcusdt@trade&quot;,&quot;btcusdt@depth@100ms&quot;]}</td></tr><tr><th>user_data_stream_path</th><td>/api/v3/userDataStream</td></tr><tr><th>api_key</th><td>NOhZ*******************</td></tr><tr><th>api_secret</th><td>yPrx*******************</td></tr><tr><th>market_config</th><td><table border=\"1\"><tr><th>price_unit</th><td>0.01</td></tr><tr><th>price_scale</th><td>2</td></tr><tr><th>size_unit</th><td>0.0001</td></tr><tr><th>size_scale</th><td>4</td></tr><tr><th>maker_fee</th><td>0.0</td></tr><tr><th>taker_fee</th><td>0.0</td></tr><tr><th>price_type</th><td>Foreign</td></tr><tr><th>fee_type</th><td>Home</td></tr><tr><th>home_currency</th><td>USDT</td></tr><tr><th>foreign_currency</th><td>BTC</td></tr></table></td></tr></table>"
       ],
       "text/plain": [
        "<IPython.core.display.HTML object>"
@@ -26932,6 +26932,59 @@
     "binance.cancel_all_orders()"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div><style>\n",
+       ".dataframe > thead > tr > th,\n",
+       ".dataframe > tbody > tr > td {\n",
+       "  text-align: right;\n",
+       "}\n",
+       "</style>\n",
+       "<small>shape: (982, 10)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>order_side</th><th>time_stamp</th><th>open</th><th>high</th><th>low</th><th>close</th><th>vol</th><th>count</th><th>start_time</th><th>end_time</th></tr><tr><td>bool</td><td>datetime[μs]</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>u32</td><td>i64</td><td>i64</td></tr></thead><tbody><tr><td>true</td><td>2023-08-30 00:00:00</td><td>27716.34</td><td>27768.56</td><td>27622.24</td><td>27622.24</td><td>958.76692</td><td>15553</td><td>1693353600001000</td><td>1693357199999000</td></tr><tr><td>true</td><td>2023-08-30 01:00:00</td><td>27622.24</td><td>27643.02</td><td>27455.91</td><td>27552.62</td><td>1115.5238</td><td>18172</td><td>1693357200000000</td><td>1693360799998000</td></tr><tr><td>true</td><td>2023-08-30 02:00:00</td><td>27552.62</td><td>27562.06</td><td>27349.39</td><td>27400.55</td><td>1303.62568</td><td>17413</td><td>1693360800009000</td><td>1693364399996000</td></tr><tr><td>true</td><td>2023-08-30 03:00:00</td><td>27400.55</td><td>27426.34</td><td>27280.99</td><td>27391.13</td><td>871.14864</td><td>13734</td><td>1693364400000000</td><td>1693367999999000</td></tr><tr><td>true</td><td>2023-08-30 04:00:00</td><td>27391.13</td><td>27446.79</td><td>27391.13</td><td>27443.0</td><td>345.27614</td><td>7657</td><td>1693368000004000</td><td>1693371599999000</td></tr><tr><td>true</td><td>2023-08-30 05:00:00</td><td>27443.0</td><td>27476.02</td><td>27435.64</td><td>27469.04</td><td>281.37395</td><td>7053</td><td>1693371600000000</td><td>1693375199997000</td></tr><tr><td>true</td><td>2023-08-30 06:00:00</td><td>27469.04</td><td>27469.04</td><td>27371.94</td><td>27382.21</td><td>498.32788</td><td>9902</td><td>1693375200006000</td><td>1693378799997000</td></tr><tr><td>true</td><td>2023-08-30 07:00:00</td><td>27382.21</td><td>27464.79</td><td>27382.21</td><td>27436.24</td><td>470.15589</td><td>10062</td><td>1693378800000000</td><td>1693382399996000</td></tr><tr><td>true</td><td>2023-08-30 08:00:00</td><td>27436.24</td><td>27498.99</td><td>27406.65</td><td>27470.97</td><td>587.05457</td><td>11005</td><td>1693382400000000</td><td>1693385999999000</td></tr><tr><td>true</td><td>2023-08-30 09:00:00</td><td>27470.97</td><td>27471.91</td><td>27310.05</td><td>27426.79</td><td>802.14641</td><td>16165</td><td>1693386000000000</td><td>1693389599991000</td></tr><tr><td>true</td><td>2023-08-30 10:00:00</td><td>27426.79</td><td>27448.85</td><td>27400.0</td><td>27435.44</td><td>282.12585</td><td>8922</td><td>1693389600004000</td><td>1693393199999000</td></tr><tr><td>true</td><td>2023-08-30 11:00:00</td><td>27435.44</td><td>27435.44</td><td>27331.45</td><td>27338.51</td><td>496.04708</td><td>12392</td><td>1693393200000000</td><td>1693396799997000</td></tr><tr><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td></tr><tr><td>false</td><td>2023-10-06 01:00:00</td><td>27457.52</td><td>27537.0</td><td>27453.58</td><td>27515.24</td><td>7.93101</td><td>1310</td><td>1696554004578000</td><td>1696557593675000</td></tr><tr><td>false</td><td>2023-10-06 02:00:00</td><td>27516.75</td><td>27589.2</td><td>27516.52</td><td>27541.64</td><td>8.65171</td><td>1258</td><td>1696557600665000</td><td>1696561185346000</td></tr><tr><td>false</td><td>2023-10-06 03:00:00</td><td>27541.64</td><td>27577.0</td><td>27535.71</td><td>27559.22</td><td>5.2508</td><td>783</td><td>1696561200574000</td><td>1696564794462000</td></tr><tr><td>false</td><td>2023-10-06 04:00:00</td><td>27559.21</td><td>27590.0</td><td>27492.11</td><td>27527.08</td><td>5.48041</td><td>860</td><td>1696564805761000</td><td>1696568398912000</td></tr><tr><td>false</td><td>2023-10-06 05:00:00</td><td>27527.86</td><td>27775.66</td><td>27353.93</td><td>27501.64</td><td>7.3563</td><td>988</td><td>1696568404570000</td><td>1696571999409000</td></tr><tr><td>false</td><td>2023-10-06 06:00:00</td><td>27500.0</td><td>27519.4</td><td>27403.75</td><td>27492.84</td><td>7.77206</td><td>995</td><td>1696572000267000</td><td>1696575417880000</td></tr><tr><td>false</td><td>2023-10-10 01:00:00</td><td>17369.33</td><td>17389.33</td><td>17322.21</td><td>17326.26</td><td>0.29971</td><td>154</td><td>1696900823578000</td><td>1696903081040000</td></tr><tr><td>false</td><td>2023-10-10 02:00:00</td><td>17329.69</td><td>27653.06</td><td>17188.05</td><td>27653.06</td><td>6.2477</td><td>872</td><td>1696903202611000</td><td>1696906735836000</td></tr><tr><td>false</td><td>2023-10-10 03:00:00</td><td>27653.06</td><td>27669.21</td><td>27633.43</td><td>27634.85</td><td>4.89965</td><td>758</td><td>1696906811023000</td><td>1696910280910000</td></tr><tr><td>false</td><td>2023-10-10 04:00:00</td><td>27632.09</td><td>27635.99</td><td>27587.68</td><td>27600.05</td><td>2.9049</td><td>564</td><td>1696910400891000</td><td>1696913942526000</td></tr><tr><td>false</td><td>2023-10-10 05:00:00</td><td>27600.05</td><td>27610.83</td><td>27579.9</td><td>27607.24</td><td>3.65201</td><td>710</td><td>1696914004349000</td><td>1696917549657000</td></tr><tr><td>false</td><td>2023-10-10 06:00:00</td><td>27607.24</td><td>27613.35</td><td>27607.24</td><td>27613.35</td><td>0.22149</td><td>35</td><td>1696917600767000</td><td>1696917630437000</td></tr></tbody></table></div>"
+      ],
+      "text/plain": [
+       "shape: (982, 10)\n",
+       "┌────────────┬────────────┬──────────┬──────────┬───┬────────────┬───────┬────────────┬────────────┐\n",
+       "│ order_side ┆ time_stamp ┆ open     ┆ high     ┆ … ┆ vol        ┆ count ┆ start_time ┆ end_time   │\n",
+       "│ ---        ┆ ---        ┆ ---      ┆ ---      ┆   ┆ ---        ┆ ---   ┆ ---        ┆ ---        │\n",
+       "│ bool       ┆ datetime[μ ┆ f64      ┆ f64      ┆   ┆ f64        ┆ u32   ┆ i64        ┆ i64        │\n",
+       "│            ┆ s]         ┆          ┆          ┆   ┆            ┆       ┆            ┆            │\n",
+       "╞════════════╪════════════╪══════════╪══════════╪═══╪════════════╪═══════╪════════════╪════════════╡\n",
+       "│ true       ┆ 2023-08-30 ┆ 27716.34 ┆ 27768.56 ┆ … ┆ 958.76692  ┆ 15553 ┆ 1693353600 ┆ 1693357199 │\n",
+       "│            ┆ 00:00:00   ┆          ┆          ┆   ┆            ┆       ┆ 001000     ┆ 999000     │\n",
+       "│ true       ┆ 2023-08-30 ┆ 27622.24 ┆ 27643.02 ┆ … ┆ 1115.5238  ┆ 18172 ┆ 1693357200 ┆ 1693360799 │\n",
+       "│            ┆ 01:00:00   ┆          ┆          ┆   ┆            ┆       ┆ 000000     ┆ 998000     │\n",
+       "│ true       ┆ 2023-08-30 ┆ 27552.62 ┆ 27562.06 ┆ … ┆ 1303.62568 ┆ 17413 ┆ 1693360800 ┆ 1693364399 │\n",
+       "│            ┆ 02:00:00   ┆          ┆          ┆   ┆            ┆       ┆ 009000     ┆ 996000     │\n",
+       "│ true       ┆ 2023-08-30 ┆ 27400.55 ┆ 27426.34 ┆ … ┆ 871.14864  ┆ 13734 ┆ 1693364400 ┆ 1693367999 │\n",
+       "│            ┆ 03:00:00   ┆          ┆          ┆   ┆            ┆       ┆ 000000     ┆ 999000     │\n",
+       "│ …          ┆ …          ┆ …        ┆ …        ┆ … ┆ …          ┆ …     ┆ …          ┆ …          │\n",
+       "│ false      ┆ 2023-10-10 ┆ 27653.06 ┆ 27669.21 ┆ … ┆ 4.89965    ┆ 758   ┆ 1696906811 ┆ 1696910280 │\n",
+       "│            ┆ 03:00:00   ┆          ┆          ┆   ┆            ┆       ┆ 023000     ┆ 910000     │\n",
+       "│ false      ┆ 2023-10-10 ┆ 27632.09 ┆ 27635.99 ┆ … ┆ 2.9049     ┆ 564   ┆ 1696910400 ┆ 1696913942 │\n",
+       "│            ┆ 04:00:00   ┆          ┆          ┆   ┆            ┆       ┆ 891000     ┆ 526000     │\n",
+       "│ false      ┆ 2023-10-10 ┆ 27600.05 ┆ 27610.83 ┆ … ┆ 3.65201    ┆ 710   ┆ 1696914004 ┆ 1696917549 │\n",
+       "│            ┆ 05:00:00   ┆          ┆          ┆   ┆            ┆       ┆ 349000     ┆ 657000     │\n",
+       "│ false      ┆ 2023-10-10 ┆ 27607.24 ┆ 27613.35 ┆ … ┆ 0.22149    ┆ 35    ┆ 1696917600 ┆ 1696917630 │\n",
+       "│            ┆ 06:00:00   ┆          ┆          ┆   ┆            ┆       ┆ 767000     ┆ 437000     │\n",
+       "└────────────┴────────────┴──────────┴──────────┴───┴────────────┴───────┴────────────┴────────────┘"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "binance.ohlcvv(0, 0, 3600)"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,