balances in

.gitignore

@@ -0,0 +1 @@
+*.ipynb

app.py

@@ -8,6 +8,8 @@ import subprocess
 import threading
 from dotenv import load_dotenv
 from requests.exceptions import ConnectionError, Timeout, TooManyRedirects
+import plotly.express as px 
+import json
 
 # ------------------------ Environment Variables --------------------------
 
@@ -60,10 +62,16 @@ def execute_cmc_scraping():
     logging.info("CMC scraping completed")
     threading.Timer(2592000 / 9000, execute_cmc_scraping).start()
 
+def execute_influencers_scraping():
+    subprocess.call(["python", "utils/scrap_influencers_balance.py"])
+    logging.info("Influencers balance scraping completed")
+    threading.Timer(3600, execute_influencers_scraping).start()  # Run every hour, for example
+
 if "initialized" not in st.session_state:
     # Start the scraping threads
     threading.Thread(target=execute_etherscan_scraping).start()
     threading.Thread(target=execute_cmc_scraping).start()
+    threading.Thread(target=execute_influencers_scraping).start()
     st.session_state["initialized"] = True
 
 #-------------------------------------streamlit ----------------------------------
@@ -167,6 +175,79 @@ with st.container():
         )
         st.plotly_chart(fig)
 
+with st.container():
+    with col1:
+
+        # Load influencers
+        def load_influencers():
+            try:
+                with open("ressources/dict_influencers_addr.json", "r") as file:
+                    return json.load(file)
+            except Exception as e:
+                st.error(f"Error loading influencers: {e}")
+                return {}
+
+        influencers = load_influencers()
+
+        new_influencer_name = st.text_input("New Influencer Name")
+        new_influencer_addr = st.text_input("New Influencer Address")
+        if st.button("Add Influencer"):
+            if new_influencer_name and new_influencer_addr:
+                influencers[new_influencer_name] = new_influencer_addr
+                with open("ressources/dict_influencers_addr.json", "w") as file:
+                    json.dump(influencers, file, indent=4)
+                st.success(f"Influencer {new_influencer_name} added")
+                subprocess.call(["python", "utils/scrap_influencers_balance.py"])
+                st.success(f"Balance updated")
+            else:
+                st.error("Please enter both name and address")
+
+        # Load Ether balances
+        try:
+            df_balances = pd.read_csv("output/influencers_balances.csv")
+            logging.info(f"Balances uploaded, shape of dataframe is {df_balances.shape}")
+            #st.write("DataFrame Loaded:", df_balances)  # Debugging line
+        except FileNotFoundError:
+            st.error("Balance data not found. Please wait for the next update cycle.")
+            df_balances = pd.DataFrame()
+            
+        # Inverting the influencers dictionary
+        inverted_influencers = {v.lower(): k for k, v in influencers.items()}
+
+        if not df_balances.empty:
+            df_balances["balance"] = df_balances["balance"].astype(float) / 1e18  # Convert Wei to Ether
+            df_balances = df_balances.rename(columns={"account": "address"})
+
+            # Ensure addresses are in the same format as in the inverted dictionary (e.g., lowercase)
+            df_balances["address"] = df_balances["address"].str.lower()
+
+            # Perform the mapping
+            df_balances["influencer"] = df_balances["address"].map(inverted_influencers)
+            #st.write("Mapped DataFrame:", df_balances)  # Debugging line
+
+            fig = px.bar(df_balances, y="influencer", x="balance",orientation="h")
+            fig.update_layout(
+                title='Ether Balances of Influencers',
+                xaxis=dict(
+                    title='Balance in eth',
+                    titlefont_size=16,
+                    tickfont_size=14,
+                ))
+            fig.update_layout(
+                autosize=False,
+                width=500,
+                height=500,
+                margin=dict(
+                    l=50,
+                    r=50,
+                    b=100,
+                    t=100,
+                    pad=4
+                ))
+
+            st.plotly_chart(fig)
+        else:
+            logging.info("DataFrame is empty")
 
 
 

app_dash.py

@@ -1,73 +0,0 @@
-import os
-import pandas as pd
-import dash
-from dash import dcc,html
-import dash_bootstrap_components as dbc
-from dash.dependencies import Input, Output
-import plotly.graph_objs as go
-
-# Load the data from the CSV files
-dataframes = []
-for filename in os.listdir('output'):
-    if filename.endswith('.csv'):
-        df = pd.read_csv(os.path.join('output', filename), sep=';')
-        dataframes.append(df)
-df = pd.concat(dataframes)
-
-
-# Create the Dash app
-app = dash.Dash(__name__, external_stylesheets=[dbc.themes.BOOTSTRAP])
-
-# Define the app layout
-app.layout = dbc.Container([
-    dbc.Row([
-        dbc.Col([
-            html.H1('Token Analysis'),
-            dcc.Dropdown(
-                id='token-dropdown',
-                options=[{'label': i, 'value': i} for i in df['tokenSymbol'].unique()],
-                value='MANA'
-            ),
-            # Add more filters here
-        ], width=5),
-        dbc.Col([
-            dcc.Graph(id='token-graph')
-        ], width=7)
-    ])
-])
-
-# Define the callback to update the graph
-@app.callback(
-    Output('token-graph', 'figure'),
-    [Input('token-dropdown', 'value')]
-)
-def update_graph(selected_token):
-    filtered_df = df[df['tokenSymbol'] == selected_token]
-    # filtered_df['timeStamp'] = pd.to_datetime(filtered_df['timeStamp'], unit='s')
-    # filtered_df['value'] = filtered_df['value'].astype(float) / 1e18
-    figure = go.Figure(
-        data=[
-            go.Scatter(
-                x=filtered_df['timeStamp'],
-                y=filtered_df['value'],
-                mode='lines',
-                name='Value over time'
-            )
-        ],
-        layout=go.Layout(
-            title='Token Value Over Time',
-            yaxis=dict(
-                title='Value ('+selected_token+')',  # Change this to 'Value (USD)' if the values are in USD
-            ),
-            showlegend=True,
-            legend=go.layout.Legend(
-                x=0,
-                y=1.0
-            ),
-            margin=go.layout.Margin(l=40, r=0, t=40, b=30)
-        )
-    )
-    return figure
-
-if __name__ == '__main__':
-    app.run_server(debug=True)

logs/scrapping/cmc_scrapping.log

@@ -176,3 +176,129 @@
 2023-11-27 23:21:54,745 [INFO] - Function fetch_and_process_cmc_data executed in 0.45 seconds
 2023-11-27 23:26:43,314 [INFO] - CMC data script execution completed.
 2023-11-27 23:26:43,315 [INFO] - Function fetch_and_process_cmc_data executed in 0.28 seconds
+2023-11-27 23:32:30,212 [INFO] - CMC data script execution completed.
+2023-11-27 23:32:30,213 [INFO] - Function fetch_and_process_cmc_data executed in 0.21 seconds
+2023-11-27 23:34:10,849 [INFO] - CMC data script execution completed.
+2023-11-27 23:34:10,850 [INFO] - Function fetch_and_process_cmc_data executed in 0.28 seconds
+2023-11-27 23:34:15,365 [INFO] - CMC data script execution completed.
+2023-11-27 23:34:15,365 [INFO] - Function fetch_and_process_cmc_data executed in 0.25 seconds
+2023-11-27 23:37:18,646 [INFO] - CMC data script execution completed.
+2023-11-27 23:37:18,647 [INFO] - Function fetch_and_process_cmc_data executed in 0.17 seconds
+2023-11-27 23:38:59,272 [INFO] - CMC data script execution completed.
+2023-11-27 23:38:59,273 [INFO] - Function fetch_and_process_cmc_data executed in 0.19 seconds
+2023-11-27 23:39:03,791 [INFO] - CMC data script execution completed.
+2023-11-27 23:39:03,792 [INFO] - Function fetch_and_process_cmc_data executed in 0.18 seconds
+2023-11-27 23:42:07,149 [INFO] - CMC data script execution completed.
+2023-11-27 23:42:07,149 [INFO] - Function fetch_and_process_cmc_data executed in 0.26 seconds
+2023-11-27 23:43:47,711 [INFO] - CMC data script execution completed.
+2023-11-27 23:43:47,712 [INFO] - Function fetch_and_process_cmc_data executed in 0.19 seconds
+2023-11-27 23:43:52,463 [INFO] - CMC data script execution completed.
+2023-11-27 23:43:52,464 [INFO] - Function fetch_and_process_cmc_data executed in 0.43 seconds
+2023-11-27 23:46:58,664 [INFO] - CMC data script execution completed.
+2023-11-27 23:46:58,664 [INFO] - Function fetch_and_process_cmc_data executed in 3.27 seconds
+2023-11-27 23:48:36,149 [INFO] - CMC data script execution completed.
+2023-11-27 23:48:36,150 [INFO] - Function fetch_and_process_cmc_data executed in 0.19 seconds
+2023-11-27 23:48:41,034 [INFO] - CMC data script execution completed.
+2023-11-27 23:48:41,035 [INFO] - Function fetch_and_process_cmc_data executed in 0.33 seconds
+2023-11-27 23:51:49,952 [INFO] - CMC data script execution completed.
+2023-11-27 23:51:49,953 [INFO] - Function fetch_and_process_cmc_data executed in 3.04 seconds
+2023-11-27 23:53:42,333 [INFO] - CMC data script execution completed.
+2023-11-27 23:53:42,334 [INFO] - Function fetch_and_process_cmc_data executed in 0.18 seconds
+2023-11-27 23:54:57,380 [INFO] - CMC data script execution completed.
+2023-11-27 23:54:57,381 [INFO] - Function fetch_and_process_cmc_data executed in 0.20 seconds
+2023-11-27 23:55:12,967 [INFO] - CMC data script execution completed.
+2023-11-27 23:55:12,968 [INFO] - Function fetch_and_process_cmc_data executed in 0.19 seconds
+2023-11-28 00:00:02,922 [INFO] - CMC data script execution completed.
+2023-11-28 00:00:02,923 [INFO] - Function fetch_and_process_cmc_data executed in 1.67 seconds
+2023-11-28 00:00:22,438 [INFO] - CMC data script execution completed.
+2023-11-28 00:00:22,439 [INFO] - Function fetch_and_process_cmc_data executed in 0.19 seconds
+2023-11-28 00:00:34,295 [INFO] - CMC data script execution completed.
+2023-11-28 00:00:34,296 [INFO] - Function fetch_and_process_cmc_data executed in 0.20 seconds
+2023-11-28 00:01:05,885 [INFO] - CMC data script execution completed.
+2023-11-28 00:01:05,886 [INFO] - Function fetch_and_process_cmc_data executed in 0.20 seconds
+2023-11-28 00:01:17,051 [INFO] - CMC data script execution completed.
+2023-11-28 00:01:17,052 [INFO] - Function fetch_and_process_cmc_data executed in 0.28 seconds
+2023-11-28 00:01:30,857 [INFO] - CMC data script execution completed.
+2023-11-28 00:01:30,858 [INFO] - Function fetch_and_process_cmc_data executed in 0.25 seconds
+2023-11-28 00:01:44,815 [INFO] - CMC data script execution completed.
+2023-11-28 00:01:44,816 [INFO] - Function fetch_and_process_cmc_data executed in 0.19 seconds
+2023-11-28 00:05:22,758 [INFO] - CMC data script execution completed.
+2023-11-28 00:05:22,759 [INFO] - Function fetch_and_process_cmc_data executed in 0.19 seconds
+2023-11-28 00:05:54,331 [INFO] - CMC data script execution completed.
+2023-11-28 00:05:54,331 [INFO] - Function fetch_and_process_cmc_data executed in 0.20 seconds
+2023-11-28 00:06:05,493 [INFO] - CMC data script execution completed.
+2023-11-28 00:06:05,494 [INFO] - Function fetch_and_process_cmc_data executed in 0.19 seconds
+2023-11-28 00:06:19,398 [INFO] - CMC data script execution completed.
+2023-11-28 00:06:19,399 [INFO] - Function fetch_and_process_cmc_data executed in 0.28 seconds
+2023-11-28 00:06:33,258 [INFO] - CMC data script execution completed.
+2023-11-28 00:06:33,258 [INFO] - Function fetch_and_process_cmc_data executed in 0.19 seconds
+2023-11-28 00:16:06,268 [INFO] - CMC data script execution completed.
+2023-11-28 00:16:06,269 [INFO] - Function fetch_and_process_cmc_data executed in 0.21 seconds
+2023-11-28 00:17:37,907 [INFO] - CMC data script execution completed.
+2023-11-28 00:17:37,908 [INFO] - Function fetch_and_process_cmc_data executed in 0.20 seconds
+2023-11-28 00:18:51,014 [INFO] - CMC data script execution completed.
+2023-11-28 00:18:51,015 [INFO] - Function fetch_and_process_cmc_data executed in 0.19 seconds
+2023-11-28 00:20:39,099 [INFO] - CMC data script execution completed.
+2023-11-28 00:20:39,100 [INFO] - Function fetch_and_process_cmc_data executed in 0.20 seconds
+2023-11-28 00:22:13,897 [INFO] - CMC data script execution completed.
+2023-11-28 00:22:13,898 [INFO] - Function fetch_and_process_cmc_data executed in 0.20 seconds
+2023-11-28 00:23:01,926 [INFO] - CMC data script execution completed.
+2023-11-28 00:23:01,927 [INFO] - Function fetch_and_process_cmc_data executed in 0.44 seconds
+2023-11-28 00:23:47,827 [INFO] - CMC data script execution completed.
+2023-11-28 00:23:47,828 [INFO] - Function fetch_and_process_cmc_data executed in 0.22 seconds
+2023-11-28 00:25:04,060 [INFO] - CMC data script execution completed.
+2023-11-28 00:25:04,061 [INFO] - Function fetch_and_process_cmc_data executed in 0.22 seconds
+2023-11-28 00:29:04,769 [INFO] - CMC data script execution completed.
+2023-11-28 00:29:04,770 [INFO] - Function fetch_and_process_cmc_data executed in 0.22 seconds
+2023-11-28 00:29:52,574 [INFO] - CMC data script execution completed.
+2023-11-28 00:29:52,575 [INFO] - Function fetch_and_process_cmc_data executed in 0.22 seconds
+2023-11-28 00:31:40,386 [INFO] - CMC data script execution completed.
+2023-11-28 00:31:40,387 [INFO] - Function fetch_and_process_cmc_data executed in 0.22 seconds
+2023-11-28 00:32:29,036 [INFO] - CMC data script execution completed.
+2023-11-28 00:32:29,037 [INFO] - Function fetch_and_process_cmc_data executed in 0.21 seconds
+2023-11-28 00:32:59,678 [INFO] - CMC data script execution completed.
+2023-11-28 00:32:59,679 [INFO] - Function fetch_and_process_cmc_data executed in 0.21 seconds
+2023-11-28 00:33:36,724 [INFO] - CMC data script execution completed.
+2023-11-28 00:33:36,725 [INFO] - Function fetch_and_process_cmc_data executed in 0.20 seconds
+2023-11-28 00:34:47,295 [INFO] - CMC data script execution completed.
+2023-11-28 00:34:47,296 [INFO] - Function fetch_and_process_cmc_data executed in 0.23 seconds
+2023-11-28 00:36:29,848 [INFO] - CMC data script execution completed.
+2023-11-28 00:36:29,849 [INFO] - Function fetch_and_process_cmc_data executed in 0.20 seconds
+2023-11-28 00:38:25,207 [INFO] - CMC data script execution completed.
+2023-11-28 00:38:25,208 [INFO] - Function fetch_and_process_cmc_data executed in 0.22 seconds
+2023-11-28 00:38:30,807 [INFO] - CMC data script execution completed.
+2023-11-28 00:38:30,808 [INFO] - Function fetch_and_process_cmc_data executed in 0.20 seconds
+2023-11-28 00:38:47,808 [INFO] - CMC data script execution completed.
+2023-11-28 00:38:47,809 [INFO] - Function fetch_and_process_cmc_data executed in 0.22 seconds
+2023-11-28 00:39:02,734 [INFO] - CMC data script execution completed.
+2023-11-28 00:39:02,735 [INFO] - Function fetch_and_process_cmc_data executed in 0.23 seconds
+2023-11-28 00:39:35,788 [INFO] - CMC data script execution completed.
+2023-11-28 00:39:35,789 [INFO] - Function fetch_and_process_cmc_data executed in 0.22 seconds
+2023-11-28 00:41:18,336 [INFO] - CMC data script execution completed.
+2023-11-28 00:41:18,337 [INFO] - Function fetch_and_process_cmc_data executed in 0.21 seconds
+2023-11-28 00:41:35,063 [INFO] - CMC data script execution completed.
+2023-11-28 00:41:35,064 [INFO] - Function fetch_and_process_cmc_data executed in 0.63 seconds
+2023-11-28 00:41:55,087 [INFO] - CMC data script execution completed.
+2023-11-28 00:41:55,088 [INFO] - Function fetch_and_process_cmc_data executed in 0.22 seconds
+2023-11-28 00:43:13,671 [INFO] - CMC data script execution completed.
+2023-11-28 00:43:13,671 [INFO] - Function fetch_and_process_cmc_data executed in 0.21 seconds
+2023-11-28 00:43:19,267 [INFO] - CMC data script execution completed.
+2023-11-28 00:43:19,268 [INFO] - Function fetch_and_process_cmc_data executed in 0.19 seconds
+2023-11-28 00:43:36,279 [INFO] - CMC data script execution completed.
+2023-11-28 00:43:36,280 [INFO] - Function fetch_and_process_cmc_data executed in 0.21 seconds
+2023-11-28 00:43:51,205 [INFO] - CMC data script execution completed.
+2023-11-28 00:43:51,206 [INFO] - Function fetch_and_process_cmc_data executed in 0.21 seconds
+2023-11-28 00:44:24,244 [INFO] - CMC data script execution completed.
+2023-11-28 00:44:24,245 [INFO] - Function fetch_and_process_cmc_data executed in 0.21 seconds
+2023-11-28 00:45:41,850 [INFO] - CMC data script execution completed.
+2023-11-28 00:45:41,851 [INFO] - Function fetch_and_process_cmc_data executed in 0.28 seconds
+2023-11-28 00:46:06,793 [INFO] - CMC data script execution completed.
+2023-11-28 00:46:06,794 [INFO] - Function fetch_and_process_cmc_data executed in 0.20 seconds
+2023-11-28 00:46:23,550 [INFO] - CMC data script execution completed.
+2023-11-28 00:46:23,551 [INFO] - Function fetch_and_process_cmc_data executed in 0.21 seconds
+2023-11-28 00:46:43,548 [INFO] - CMC data script execution completed.
+2023-11-28 00:46:43,549 [INFO] - Function fetch_and_process_cmc_data executed in 0.20 seconds
+2023-11-28 00:48:02,131 [INFO] - CMC data script execution completed.
+2023-11-28 00:48:02,132 [INFO] - Function fetch_and_process_cmc_data executed in 0.21 seconds
+2023-11-28 00:48:07,702 [INFO] - CMC data script execution completed.
+2023-11-28 00:48:07,703 [INFO] - Function fetch_and_process_cmc_data executed in 0.19 seconds

logs/scrapping/etherscan_scrap.log

@@ -83,3 +83,123 @@
 2023-11-27 23:22:15,713 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
 2023-11-27 23:22:15,713 [INFO] - Etherscan scraping script execution completed.
 2023-11-27 23:22:15,713 [INFO] - Function fetch_and_update_etherscan executed in 21.43 seconds
+2023-11-27 23:32:52,250 [INFO] - Created files: 
+2023-11-27 23:32:52,250 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-27 23:32:52,250 [INFO] - Etherscan scraping script execution completed.
+2023-11-27 23:32:52,250 [INFO] - Function fetch_and_update_etherscan executed in 22.27 seconds
+2023-11-27 23:34:33,682 [INFO] - Created files: 
+2023-11-27 23:34:33,682 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-27 23:34:33,682 [INFO] - Etherscan scraping script execution completed.
+2023-11-27 23:34:33,682 [INFO] - Function fetch_and_update_etherscan executed in 23.10 seconds
+2023-11-27 23:34:37,126 [INFO] - Created files: 
+2023-11-27 23:34:37,126 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-27 23:34:37,126 [INFO] - Etherscan scraping script execution completed.
+2023-11-27 23:34:37,126 [INFO] - Function fetch_and_update_etherscan executed in 22.01 seconds
+2023-11-27 23:54:04,485 [INFO] - Created files: 
+2023-11-27 23:54:04,485 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-27 23:54:04,485 [INFO] - Etherscan scraping script execution completed.
+2023-11-27 23:54:04,486 [INFO] - Function fetch_and_update_etherscan executed in 22.33 seconds
+2023-11-27 23:55:33,934 [INFO] - Created files: 
+2023-11-27 23:55:33,934 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-27 23:55:33,934 [INFO] - Etherscan scraping script execution completed.
+2023-11-27 23:55:33,934 [INFO] - Function fetch_and_update_etherscan executed in 21.16 seconds
+2023-11-28 00:00:56,044 [INFO] - Created files: 
+2023-11-28 00:00:56,044 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 00:00:56,044 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 00:00:56,044 [INFO] - Function fetch_and_update_etherscan executed in 21.94 seconds
+2023-11-28 00:01:27,862 [INFO] - Created files: 
+2023-11-28 00:01:27,862 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 00:01:27,862 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 00:01:27,863 [INFO] - Function fetch_and_update_etherscan executed in 22.18 seconds
+2023-11-28 00:01:42,242 [INFO] - Created files: 
+2023-11-28 00:01:42,242 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 00:01:42,242 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 00:01:42,242 [INFO] - Function fetch_and_update_etherscan executed in 25.47 seconds
+2023-11-28 00:02:02,570 [INFO] - Created files: 
+2023-11-28 00:02:02,571 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 00:02:02,571 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 00:02:02,571 [INFO] - Function fetch_and_update_etherscan executed in 31.96 seconds
+2023-11-28 00:02:09,587 [INFO] - Created files: 
+2023-11-28 00:02:09,587 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 00:02:09,587 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 00:02:09,587 [INFO] - Function fetch_and_update_etherscan executed in 24.99 seconds
+2023-11-28 00:16:29,245 [INFO] - Created files: 
+2023-11-28 00:16:29,245 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 00:16:29,245 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 00:16:29,245 [INFO] - Function fetch_and_update_etherscan executed in 23.22 seconds
+2023-11-28 00:17:59,714 [INFO] - Created files: 
+2023-11-28 00:17:59,714 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 00:17:59,714 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 00:17:59,714 [INFO] - Function fetch_and_update_etherscan executed in 22.03 seconds
+2023-11-28 00:19:16,916 [INFO] - Created files: 
+2023-11-28 00:19:16,916 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 00:19:16,916 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 00:19:16,916 [INFO] - Function fetch_and_update_etherscan executed in 26.08 seconds
+2023-11-28 00:21:02,924 [INFO] - Created files: 
+2023-11-28 00:21:02,924 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 00:21:02,924 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 00:21:02,924 [INFO] - Function fetch_and_update_etherscan executed in 24.03 seconds
+2023-11-28 00:23:27,103 [INFO] - Created files: 
+2023-11-28 00:23:27,103 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 00:23:27,103 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 00:23:27,103 [INFO] - Function fetch_and_update_etherscan executed in 25.61 seconds
+2023-11-28 00:24:09,758 [INFO] - Created files: 
+2023-11-28 00:24:09,758 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 00:24:09,759 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 00:24:09,759 [INFO] - Function fetch_and_update_etherscan executed in 22.12 seconds
+2023-11-28 00:25:24,884 [INFO] - Created files: 
+2023-11-28 00:25:24,884 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 00:25:24,884 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 00:25:24,884 [INFO] - Function fetch_and_update_etherscan executed in 21.00 seconds
+2023-11-28 00:29:25,777 [INFO] - Created files: 
+2023-11-28 00:29:25,777 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 00:29:25,777 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 00:29:25,777 [INFO] - Function fetch_and_update_etherscan executed in 21.20 seconds
+2023-11-28 00:32:02,225 [INFO] - Created files: 
+2023-11-28 00:32:02,225 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 00:32:02,225 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 00:32:02,225 [INFO] - Function fetch_and_update_etherscan executed in 22.06 seconds
+2023-11-28 00:32:49,987 [INFO] - Created files: 
+2023-11-28 00:32:49,987 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 00:32:49,987 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 00:32:49,987 [INFO] - Function fetch_and_update_etherscan executed in 21.20 seconds
+2023-11-28 00:33:23,088 [INFO] - Created files: 
+2023-11-28 00:33:23,088 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 00:33:23,088 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 00:33:23,088 [INFO] - Function fetch_and_update_etherscan executed in 23.61 seconds
+2023-11-28 00:33:57,499 [INFO] - Created files: 
+2023-11-28 00:33:57,500 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 00:33:57,500 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 00:33:57,500 [INFO] - Function fetch_and_update_etherscan executed in 20.97 seconds
+2023-11-28 00:35:08,107 [INFO] - Created files: 
+2023-11-28 00:35:08,108 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 00:35:08,108 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 00:35:08,108 [INFO] - Function fetch_and_update_etherscan executed in 21.05 seconds
+2023-11-28 00:36:52,656 [INFO] - Created files: 
+2023-11-28 00:36:52,656 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 00:36:52,656 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 00:36:52,656 [INFO] - Function fetch_and_update_etherscan executed in 23.00 seconds
+2023-11-28 00:38:51,771 [INFO] - Created files: 
+2023-11-28 00:38:51,771 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 00:38:51,771 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 00:38:51,771 [INFO] - Function fetch_and_update_etherscan executed in 21.18 seconds
+2023-11-28 00:39:11,600 [INFO] - Created files: 
+2023-11-28 00:39:11,601 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 00:39:11,601 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 00:39:11,601 [INFO] - Function fetch_and_update_etherscan executed in 24.03 seconds
+2023-11-28 00:39:25,463 [INFO] - Created files: 
+2023-11-28 00:39:25,464 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 00:39:25,464 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 00:39:25,464 [INFO] - Function fetch_and_update_etherscan executed in 22.97 seconds
+2023-11-28 00:41:57,391 [INFO] - Created files: 
+2023-11-28 00:41:57,392 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 00:41:57,392 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 00:41:57,392 [INFO] - Function fetch_and_update_etherscan executed in 22.96 seconds
+2023-11-28 00:42:21,289 [INFO] - Created files: 
+2023-11-28 00:42:21,289 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 00:42:21,289 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 00:42:21,289 [INFO] - Function fetch_and_update_etherscan executed in 26.43 seconds
+2023-11-28 00:46:02,648 [INFO] - Created files: 
+2023-11-28 00:46:02,648 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 00:46:02,648 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 00:46:02,648 [INFO] - Function fetch_and_update_etherscan executed in 21.07 seconds

logs/streamlit/front.log

@@ -176,3 +176,196 @@
 2023-11-27 23:21:54,811 [INFO] - CMC scraping completed
 2023-11-27 23:22:15,775 [INFO] - Etherscan scraping completed
 2023-11-27 23:26:43,359 [INFO] - CMC scraping completed
+2023-11-27 23:32:29,592 [INFO] - Streamlit app has started
+2023-11-27 23:32:30,264 [INFO] - CMC scraping completed
+2023-11-27 23:32:34,138 [INFO] - Streamlit app has started
+2023-11-27 23:32:37,313 [INFO] - Streamlit app has started
+2023-11-27 23:32:39,895 [INFO] - Streamlit app has started
+2023-11-27 23:32:52,287 [INFO] - Etherscan scraping completed
+2023-11-27 23:32:55,329 [INFO] - Streamlit app has started
+2023-11-27 23:32:57,613 [INFO] - Streamlit app has started
+2023-11-27 23:34:01,964 [INFO] - Streamlit app has started
+2023-11-27 23:34:04,098 [INFO] - Streamlit app has started
+2023-11-27 23:34:10,215 [INFO] - Streamlit app has started
+2023-11-27 23:34:10,882 [INFO] - CMC scraping completed
+2023-11-27 23:34:13,138 [INFO] - Streamlit app has started
+2023-11-27 23:34:14,760 [INFO] - Streamlit app has started
+2023-11-27 23:34:15,400 [INFO] - CMC scraping completed
+2023-11-27 23:34:33,717 [INFO] - Etherscan scraping completed
+2023-11-27 23:34:37,162 [INFO] - Etherscan scraping completed
+2023-11-27 23:37:18,685 [INFO] - CMC scraping completed
+2023-11-27 23:38:59,315 [INFO] - CMC scraping completed
+2023-11-27 23:39:03,833 [INFO] - CMC scraping completed
+2023-11-27 23:42:07,188 [INFO] - CMC scraping completed
+2023-11-27 23:43:47,753 [INFO] - CMC scraping completed
+2023-11-27 23:43:52,502 [INFO] - CMC scraping completed
+2023-11-27 23:46:58,703 [INFO] - CMC scraping completed
+2023-11-27 23:48:36,187 [INFO] - CMC scraping completed
+2023-11-27 23:48:41,072 [INFO] - CMC scraping completed
+2023-11-27 23:51:49,990 [INFO] - CMC scraping completed
+2023-11-27 23:53:41,798 [INFO] - Streamlit app has started
+2023-11-27 23:53:42,395 [INFO] - CMC scraping completed
+2023-11-27 23:54:04,552 [INFO] - Etherscan scraping completed
+2023-11-27 23:54:56,822 [INFO] - Streamlit app has started
+2023-11-27 23:54:57,423 [INFO] - CMC scraping completed
+2023-11-27 23:55:10,329 [INFO] - Etherscan scraping completed
+2023-11-27 23:55:12,373 [INFO] - Streamlit app has started
+2023-11-27 23:55:13,009 [INFO] - CMC scraping completed
+2023-11-27 23:55:33,990 [INFO] - Etherscan scraping completed
+2023-11-28 00:00:02,968 [INFO] - CMC scraping completed
+2023-11-28 00:00:21,807 [INFO] - Streamlit app has started
+2023-11-28 00:00:22,345 [INFO] - Influencers balance scraping completed
+2023-11-28 00:00:22,479 [INFO] - CMC scraping completed
+2023-11-28 00:00:32,165 [INFO] - Etherscan scraping completed
+2023-11-28 00:00:33,656 [INFO] - Streamlit app has started
+2023-11-28 00:00:34,195 [INFO] - Influencers balance scraping completed
+2023-11-28 00:00:34,368 [INFO] - CMC scraping completed
+2023-11-28 00:00:56,085 [INFO] - Etherscan scraping completed
+2023-11-28 00:01:05,297 [INFO] - Streamlit app has started
+2023-11-28 00:01:05,750 [INFO] - Influencers balance scraping completed
+2023-11-28 00:01:05,921 [INFO] - CMC scraping completed
+2023-11-28 00:01:16,398 [INFO] - Streamlit app has started
+2023-11-28 00:01:16,845 [INFO] - Influencers balance scraping completed
+2023-11-28 00:01:17,106 [INFO] - CMC scraping completed
+2023-11-28 00:01:27,905 [INFO] - Etherscan scraping completed
+2023-11-28 00:01:30,236 [INFO] - Streamlit app has started
+2023-11-28 00:01:30,656 [INFO] - Influencers balance scraping completed
+2023-11-28 00:01:30,913 [INFO] - CMC scraping completed
+2023-11-28 00:01:42,295 [INFO] - Etherscan scraping completed
+2023-11-28 00:01:44,168 [INFO] - Streamlit app has started
+2023-11-28 00:01:44,643 [INFO] - Influencers balance scraping completed
+2023-11-28 00:01:44,866 [INFO] - CMC scraping completed
+2023-11-28 00:02:02,613 [INFO] - Etherscan scraping completed
+2023-11-28 00:02:09,636 [INFO] - Etherscan scraping completed
+2023-11-28 00:05:22,793 [INFO] - CMC scraping completed
+2023-11-28 00:05:54,368 [INFO] - CMC scraping completed
+2023-11-28 00:06:05,531 [INFO] - CMC scraping completed
+2023-11-28 00:06:19,440 [INFO] - CMC scraping completed
+2023-11-28 00:06:33,296 [INFO] - CMC scraping completed
+2023-11-28 00:16:05,661 [INFO] - Streamlit app has started
+2023-11-28 00:16:06,310 [INFO] - CMC scraping completed
+2023-11-28 00:16:06,587 [INFO] - Influencers balance scraping completed
+2023-11-28 00:16:29,280 [INFO] - Etherscan scraping completed
+2023-11-28 00:17:37,302 [INFO] - Streamlit app has started
+2023-11-28 00:17:37,959 [INFO] - CMC scraping completed
+2023-11-28 00:17:38,217 [INFO] - Influencers balance scraping completed
+2023-11-28 00:17:59,750 [INFO] - Etherscan scraping completed
+2023-11-28 00:18:50,436 [INFO] - Streamlit app has started
+2023-11-28 00:18:50,555 [INFO] - balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 00:18:51,082 [INFO] - CMC scraping completed
+2023-11-28 00:18:51,352 [INFO] - Influencers balance scraping completed
+2023-11-28 00:19:16,960 [INFO] - Etherscan scraping completed
+2023-11-28 00:20:38,482 [INFO] - Streamlit app has started
+2023-11-28 00:20:38,863 [INFO] - balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 00:20:38,863 [INFO] - Balance is not empty
+2023-11-28 00:20:39,138 [INFO] - CMC scraping completed
+2023-11-28 00:20:39,434 [INFO] - Influencers balance scraping completed
+2023-11-28 00:21:02,960 [INFO] - Etherscan scraping completed
+2023-11-28 00:22:13,322 [INFO] - Streamlit app has started
+2023-11-28 00:22:13,439 [INFO] - balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 00:22:13,439 [INFO] - Balance is not empty
+2023-11-28 00:22:13,958 [INFO] - CMC scraping completed
+2023-11-28 00:22:14,242 [INFO] - Influencers balance scraping completed
+2023-11-28 00:22:59,094 [INFO] - Etherscan scraping completed
+2023-11-28 00:23:01,085 [INFO] - Streamlit app has started
+2023-11-28 00:23:01,482 [INFO] - balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 00:23:01,482 [INFO] - Balance is not empty
+2023-11-28 00:23:01,975 [INFO] - CMC scraping completed
+2023-11-28 00:23:05,581 [INFO] - Influencers balance scraping completed
+2023-11-28 00:23:27,142 [INFO] - Etherscan scraping completed
+2023-11-28 00:23:47,186 [INFO] - Streamlit app has started
+2023-11-28 00:23:47,327 [INFO] - balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 00:23:47,328 [INFO] - Balance is not empty
+2023-11-28 00:23:47,880 [INFO] - CMC scraping completed
+2023-11-28 00:23:48,138 [INFO] - Influencers balance scraping completed
+2023-11-28 00:24:09,799 [INFO] - Etherscan scraping completed
+2023-11-28 00:25:03,453 [INFO] - Streamlit app has started
+2023-11-28 00:25:03,851 [INFO] - balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 00:25:03,851 [INFO] - Balance is not empty
+2023-11-28 00:25:04,099 [INFO] - CMC scraping completed
+2023-11-28 00:25:04,392 [INFO] - Influencers balance scraping completed
+2023-11-28 00:25:24,922 [INFO] - Etherscan scraping completed
+2023-11-28 00:29:04,154 [INFO] - Streamlit app has started
+2023-11-28 00:29:04,280 [INFO] - Balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 00:29:04,818 [INFO] - CMC scraping completed
+2023-11-28 00:29:05,098 [INFO] - Influencers balance scraping completed
+2023-11-28 00:29:25,819 [INFO] - Etherscan scraping completed
+2023-11-28 00:29:52,624 [INFO] - CMC scraping completed
+2023-11-28 00:31:39,706 [INFO] - Streamlit app has started
+2023-11-28 00:31:39,828 [INFO] - Balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 00:31:40,455 [INFO] - CMC scraping completed
+2023-11-28 00:31:40,706 [INFO] - Influencers balance scraping completed
+2023-11-28 00:32:02,270 [INFO] - Etherscan scraping completed
+2023-11-28 00:32:28,378 [INFO] - Streamlit app has started
+2023-11-28 00:32:28,800 [INFO] - Balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 00:32:29,086 [INFO] - CMC scraping completed
+2023-11-28 00:32:29,323 [INFO] - Influencers balance scraping completed
+2023-11-28 00:32:50,023 [INFO] - Etherscan scraping completed
+2023-11-28 00:32:59,053 [INFO] - Streamlit app has started
+2023-11-28 00:32:59,440 [INFO] - Balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 00:32:59,760 [INFO] - CMC scraping completed
+2023-11-28 00:33:00,008 [INFO] - Influencers balance scraping completed
+2023-11-28 00:33:23,128 [INFO] - Etherscan scraping completed
+2023-11-28 00:33:36,115 [INFO] - Streamlit app has started
+2023-11-28 00:33:36,510 [INFO] - Balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 00:33:36,789 [INFO] - CMC scraping completed
+2023-11-28 00:33:37,091 [INFO] - Influencers balance scraping completed
+2023-11-28 00:33:57,538 [INFO] - Etherscan scraping completed
+2023-11-28 00:34:46,665 [INFO] - Streamlit app has started
+2023-11-28 00:34:46,786 [INFO] - Balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 00:34:47,345 [INFO] - CMC scraping completed
+2023-11-28 00:34:47,615 [INFO] - Influencers balance scraping completed
+2023-11-28 00:35:08,147 [INFO] - Etherscan scraping completed
+2023-11-28 00:36:29,251 [INFO] - Streamlit app has started
+2023-11-28 00:36:29,364 [INFO] - Balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 00:36:29,906 [INFO] - CMC scraping completed
+2023-11-28 00:36:30,223 [INFO] - Influencers balance scraping completed
+2023-11-28 00:36:52,694 [INFO] - Etherscan scraping completed
+2023-11-28 00:38:25,244 [INFO] - CMC scraping completed
+2023-11-28 00:38:30,209 [INFO] - Streamlit app has started
+2023-11-28 00:38:30,325 [INFO] - Balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 00:38:30,873 [INFO] - CMC scraping completed
+2023-11-28 00:38:31,159 [INFO] - Influencers balance scraping completed
+2023-11-28 00:38:47,173 [INFO] - Streamlit app has started
+2023-11-28 00:38:47,297 [INFO] - Balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 00:38:47,855 [INFO] - CMC scraping completed
+2023-11-28 00:38:48,075 [INFO] - Influencers balance scraping completed
+2023-11-28 00:38:51,818 [INFO] - Etherscan scraping completed
+2023-11-28 00:39:02,096 [INFO] - Streamlit app has started
+2023-11-28 00:39:02,213 [INFO] - Balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 00:39:02,789 [INFO] - CMC scraping completed
+2023-11-28 00:39:03,019 [INFO] - Influencers balance scraping completed
+2023-11-28 00:39:11,636 [INFO] - Etherscan scraping completed
+2023-11-28 00:39:25,504 [INFO] - Etherscan scraping completed
+2023-11-28 00:39:35,829 [INFO] - CMC scraping completed
+2023-11-28 00:41:18,387 [INFO] - CMC scraping completed
+2023-11-28 00:41:34,014 [INFO] - Streamlit app has started
+2023-11-28 00:41:34,141 [INFO] - Balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 00:41:34,945 [INFO] - Influencers balance scraping completed
+2023-11-28 00:41:35,102 [INFO] - CMC scraping completed
+2023-11-28 00:41:54,448 [INFO] - Streamlit app has started
+2023-11-28 00:41:54,587 [INFO] - Balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 00:41:55,142 [INFO] - CMC scraping completed
+2023-11-28 00:41:55,379 [INFO] - Influencers balance scraping completed
+2023-11-28 00:41:57,427 [INFO] - Etherscan scraping completed
+2023-11-28 00:42:21,330 [INFO] - Etherscan scraping completed
+2023-11-28 00:43:13,716 [INFO] - CMC scraping completed
+2023-11-28 00:43:19,307 [INFO] - CMC scraping completed
+2023-11-28 00:43:36,329 [INFO] - CMC scraping completed
+2023-11-28 00:43:51,243 [INFO] - CMC scraping completed
+2023-11-28 00:44:24,288 [INFO] - CMC scraping completed
+2023-11-28 00:45:41,175 [INFO] - Streamlit app has started
+2023-11-28 00:45:41,291 [INFO] - Balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 00:45:41,896 [INFO] - CMC scraping completed
+2023-11-28 00:45:42,150 [INFO] - Influencers balance scraping completed
+2023-11-28 00:46:02,684 [INFO] - Etherscan scraping completed
+2023-11-28 00:46:06,833 [INFO] - CMC scraping completed
+2023-11-28 00:46:23,599 [INFO] - CMC scraping completed
+2023-11-28 00:46:43,588 [INFO] - CMC scraping completed
+2023-11-28 00:47:17,582 [INFO] - Streamlit app has started
+2023-11-28 00:47:17,698 [INFO] - Balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 00:47:27,768 [INFO] - Streamlit app has started
+2023-11-28 00:47:27,856 [INFO] - Streamlit app has started
+2023-11-28 00:47:28,807 [INFO] - Balances uploaded, shape of dataframe is (4, 2)
+2023-11-28 00:48:02,170 [INFO] - CMC scraping completed
+2023-11-28 00:48:07,741 [INFO] - CMC scraping completed

output/cmc_data_v1_cryptocurrency_listings_latest_100.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f6565d58fdcb4f22b4ba93cb879ca3c74d71096a5b1ed911e625c35ae56432a0
-size 119080
+oid sha256:5be145c802161fd6ec2f4932bd5c782d41e2c1e166a966bdaf080955e45397eb
+size 119005

output/influencers_balances.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a132619b2c97f10e5a84b873ea52f108b2a9e61d1f18f8d2aeaf4140a22b7bc
+size 247

output/top_100_update.csv

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:767a255420ffd1a3e5480673610f1ca21a10b9c2a9f402d8a421212f5e63270e
-size 129574
+oid sha256:7e6bafeb039ce6f24f145e6ebbec3eacbec576d432111ce9767686e3a85ccdf3
+size 748449

output/transactions_APE.csv

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1fe609b08835b469af3f98a03959a75ecaaa119d50d222bd95981117cce95a56
-size 3430047
+oid sha256:fd20aeb6fc41c40b6d62bf00c3921bfa5cc0c803aea1a2817656db5efdebf81c
+size 3457395

output/transactions_AXIE.csv

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:497efc710a52dfb17f9dacc50500293cde3450644a84069fbdd4c1cae8462b37
-size 1586211
+oid sha256:e1bcc9db49c214ca61d447a783162785e1e3a2e11cc5b037e8430addfbfd42d9
+size 1610374

output/transactions_GALA.csv

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dbdcc912f4193543018955be769a5418e98ab8dd3ebfaec7e47b290ac7c74f5d
-size 4094956
+oid sha256:3c382d5715b1551fe22176130dd9b139d446339819cfe22eb8e55cbdaf391858
+size 4164007

output/transactions_MANA.csv

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d530a0dd8a8f62f11f8c912b0f90cccc849349fa743873d05fbc5c14254fa84b
-size 2223053
+oid sha256:954bb98d20c48c0e62cb3349dbaeb4af1499c5738f664f4434cd170d9351b787
+size 2237025

ressources/dict_influencers_addr.json

@@ -1,4 +1,6 @@
-{"alexBecker":"0x4d224452801aced8b2f0aebe155379bb5d594381",
-"elliotrades":"0xc02f318365582557a482eb5d1834dfd7af4a3f59",
-"jrny":"0x08c1ae7e46d4a13b766566033b5c47c735e19f6f"
-}
+{
+    "alexBecker": "0x4d224452801aced8b2f0aebe155379bb5d594381",
+    "elliotrades": "0xc02f318365582557a482eb5d1834dfd7af4a3f59",
+    "jrny": "0x08c1ae7e46d4a13b766566033b5c47c735e19f6f",
+    "momo": "0x17770D4A8C0d068be7eB5b7AF453f9E176E52464"
+}

scrap_data_cmc.py

@@ -1,90 +0,0 @@
-#-------------------------------------libraries ----------------------------------
-
-from requests import Request, Session
-from requests.exceptions import ConnectionError, Timeout, TooManyRedirects
-import json
-import os
-import pandas as pd
-import numpy as np
-import logging
-from dotenv import load_dotenv
-load_dotenv()
-
-#-------------------------------------env vars----------------------------------
-
-url = os.getenv("URL_CMC")
-endpoints = ["v1/cryptocurrency/listings/latest",
-             #"/v1/cryptocurrency/trending/latest",
-             ]
-start = "1"
-stop = "100"
-parameters = {
-  'start':start,
-  'limit':stop,
-  'convert':'USD'
-}
-headers = {
-  'Accepts': 'application/json',
-  'X-CMC_PRO_API_KEY': os.getenv("API_KEY_CMC"),
-}
-
-# Configure the logging settings
-log_folder = "./logs/scrapping/"
-os.makedirs(log_folder, exist_ok=True)  # Ensure the log folder exists
-log_file = os.path.join(log_folder, "scrapping.log")
-log_format = "%(asctime)s [%(levelname)s] - %(message)s"
-logging.basicConfig(filename=log_file, level=logging.INFO, format=log_format)
-
-#-------------------------------------api call----------------------------------
-
-session = Session()
-session.headers.update(headers)
-
-for endpoint in endpoints:
-  target = f"{url}/{endpoint}"
-  try:
-      response = session.get(target, params=parameters)
-      data = json.loads(response.text)
-      with open(f'output/cmc_data_{endpoint.replace("/", "_")}_{stop}.json', 'w') as f:
-          json.dump(data, f)
-      logging.info(f"Successfully fetched data from {target}")
-  except (ConnectionError, Timeout, TooManyRedirects) as e:
-      logging.error(f"Error while fetching data from {target}: {e}")
-
-#-------------------------------------process data----------------------------------
-
-# create data frame with chosen columns  
-df = pd.DataFrame(data["data"])[["name","symbol","circulating_supply","total_supply","quote"]]
-# explode column quote then chose columns 
-quote_df = pd.json_normalize(df['quote'].apply(lambda x: x['USD']))[["price","percent_change_24h","percent_change_7d","percent_change_90d","market_cap","fully_diluted_market_cap","last_updated"]]
-# drop quote
-df = df.drop("quote",axis=1)
-# create features
-df["percent_tokens_circulation"] = np.round((df["circulating_supply"]/df["total_supply"])*100,1)
-# merge dataframe
-df = df.join(quote_df)
-df["last_updated"] = pd.to_datetime(df["last_updated"])
-df.to_csv(f"output/top_{stop}_update.csv")
-
-#-------------------------------------save data----------------------------------
-
-# Check if the file exists
-output_file = f"output/top_{stop}_update.csv"
-if os.path.isfile(output_file):
-    logging.info("Updating dataset"+f"top_{stop}_update"+". ")
-    # Read the existing data
-    existing_data = pd.read_csv(output_file)
-    # Concatenate the existing data with the new data vertically
-    updated_data = pd.concat([existing_data, df], axis=0, ignore_index=True)
-    # Remove duplicates (if any) based on a unique identifier column
-    updated_data.drop_duplicates(subset=["symbol", "last_updated"], inplace=True)
-    # Save the updated data back to the same file
-    updated_data.to_csv(output_file, index=False)
-else:
-    # If the file doesn't exist, save the current data to it
-    df.to_csv(output_file, index=False)
-logging.info("Script execution completed.")
-
-
-
-#-------------------------------------end----------------------------------

scrap_data_etherscan.py

@@ -1,17 +0,0 @@
-import requests
-import time
-import pandas as pd
-import json
-import os
-from utils.scrap import update_and_save_csv
-
-# Create output folder
-if not os.path.exists("output"):
-    os.makedirs("output")
-
-# Load the JSON file into a dictionary
-print(os.getcwd())
-with open("ressources/dict_tokens_addr.json", "r") as file:
-    dict_addresses = json.load(file)
-
-update_and_save_csv(dict_addresses)

utils/scrap.py

@@ -1,198 +0,0 @@
-# ---------------------- Library Imports ----------------------
-import time
-import os
-import json
-import pandas as pd
-import numpy as np
-import logging
-import requests
-from dotenv import load_dotenv
-from requests import Session
-
-# ---------------------- Environment Variables ----------------------
-
-load_dotenv()
-# Etherscan API
-url_etherscan = os.getenv("URL_ETHERSCAN")
-api_key_etherscan = os.getenv("API_KEY_ETHERSCAN")
-
-# CoinMarketCap API
-url_cmc = os.getenv("URL_CMC")
-api_key_cmc = os.getenv("API_KEY_CMC")
-
-# Logging
-log_folder = os.getenv("LOG_FOLDER")
-os.makedirs(log_folder, exist_ok=True)
-log_file = os.path.join(log_folder, "scrapping.log")
-log_format = "%(asctime)s [%(levelname)s] - %(message)s"
-logging.basicConfig(filename=log_file, level=logging.INFO, format=log_format)
-
-# Load the JSON file into a dictionary
-with open("ressources/dict_tokens_addr.json", "r") as file:
-    dict_addresses = json.load(file)
-
-L_created = []
-L_updated = []
-
-# Define the number of blocks to retrieve transactions from
-n_blocks = 20000
-n_loop = n_blocks // 10_000
-
-# ---------------------- Processing ----------------------
-
-# Helper function for logging execution time
-def log_execution_time(func):
-    def wrapper(*args, **kwargs):
-        start_time = time.time()
-        result = func(*args, **kwargs)
-        end_time = time.time()
-        logging.info(f"Function {func.__name__} executed in {end_time - start_time:.2f} seconds")
-        return result
-    return wrapper
-
-
-@log_execution_time
-# function 1: Fetch and Update Etherscan Data
-def fetch_and_update_etherscan():
-    for tokenSymbol, contractAddr in dict_addresses.items():
-        file = f"output/transactions_{tokenSymbol}.csv"
-        if not os.path.exists(file):
-            L_created.append(file)
-            df_transactions = get_coin_data(contractAddr, n_loop)
-            df_transactions_no_dup = df_transactions.drop(["confirmations", "input"], axis=1).drop_duplicates(subset="hash")
-            df_transactions_no_dup.to_csv(file, sep=",", index=False)
-        else:
-            L_updated.append(file)
-            df_temp = pd.read_csv(file, sep=",")
-            df_temp = df_temp.sort_values("blockNumber", ascending=False)
-            start_block = df_temp["blockNumber"].iloc[0]
-
-            # Retrieve latest block number and calculate the difference
-            latest_block_number, diff = latest_block(start_block)
-            if latest_block_number is None:
-                logging.error(f"Failed to retrieve latest block number for token: {tokenSymbol}")
-                continue  # Skip to the next token if the latest block number could not be retrieved
-
-            n_loop_to_concat = (diff // 10000) + 1
-            df_transactions = get_coin_data(contractAddr, n_loop_to_concat)
-            df_latest = pd.concat([df_transactions, df_temp]).drop(["confirmations", "input"], axis=1)
-            df_latest_no_dup = df_latest.drop_duplicates(subset="hash")
-            df_latest_no_dup.loc[:, "blockNumber"] = df_latest_no_dup["blockNumber"].astype(int)
-            df_latest_no_dup = df_latest_no_dup.sort_values(by="blockNumber")
-            df_latest_no_dup.to_csv(file, sep=",", index=False)
-
-    logging.info("Created files: " + ", ".join(L_created))
-    logging.info("Updated files: " + ", ".join(L_updated))
-    logging.info("Script execution completed.")
-
-# Helper function to get latest block number
-def latest_block(start_block=None):
-    params = {
-        "module": "proxy",
-        "action": "eth_blockNumber",
-        "apikey": api_key_etherscan
-    }
-    response = requests.get(url_etherscan, params=params)
-    if response.status_code == 200:
-        try:
-            latest_block_number = int(response.json()["result"], 16)
-            if start_block is not None:
-                return latest_block_number, latest_block_number - start_block
-            return latest_block_number
-        except (ValueError, KeyError):
-            logging.error(f"Invalid response format or missing data in response: {response.json()}")
-            return None, None
-    else:
-        logging.error(f"API call failed with status code {response.status_code}: {response.json()}")
-        return None, None
-
-def get_coin_data(contractAddr, n):
-    latest_block_number = latest_block()
-    if latest_block_number is None:
-        logging.error(f"Could not retrieve latest block number for contract address {contractAddr}")
-        return pd.DataFrame()  # Return an empty DataFrame
-
-    df_transactions = pd.DataFrame()
-    transactions_per_call = 10_000
-    for i in range(n):
-        start_block = latest_block_number - (n - i) * transactions_per_call
-        end_block = latest_block_number - (n - 1 - i) * transactions_per_call
-        params = {
-            "module": "account",
-            "action": "tokentx",
-            "contractaddress": contractAddr,
-            "startblock": start_block,
-            "endblock": end_block,
-            "sort": "asc",
-            "apikey": api_key_etherscan
-        }
-        response = requests.get(url_etherscan, params=params)
-        transactions = response.json().get("result", [])
-        
-        # Check if transactions is a list of dictionaries
-        if not isinstance(transactions, list) or not all(isinstance(item, dict) for item in transactions):
-            logging.error(f"Invalid data format for transactions: {transactions}")
-            continue  # Skip this iteration if transactions data is invalid
-
-        df_temp = pd.DataFrame(transactions)
-        if not df_temp.empty:
-            df_transactions = pd.concat([df_transactions, df_temp])
-        time.sleep(1)
-    
-    if 'timeStamp' in df_transactions:
-        df_transactions['timeStamp'] = pd.to_datetime(df_transactions['timeStamp'].astype(int), unit='s')
-    else:
-        logging.error("'timeStamp' key not found in the response data.")
-        return pd.DataFrame()  # Return an empty DataFrame if key is missing
-    df_transactions['value'] = df_transactions['value'].astype(float) / 1e18
-    return df_transactions
-
-# function 2: Fetch and Process CMC Data
-@log_execution_time
-def fetch_and_process_cmc_data():
-    session = Session()
-    session.headers.update({
-        'Accepts': 'application/json',
-        'X-CMC_PRO_API_KEY': api_key_cmc,
-    })
-    parameters = {
-        'start': '1',
-        'limit': '100',
-        'convert': 'USD'
-    }
-
-    for endpoint in ["v1/cryptocurrency/listings/latest"]:
-        target = f"{url_cmc}/{endpoint}"
-        try:
-            response = session.get(target, params=parameters)
-            data = json.loads(response.text)
-            with open(f'output/cmc_data_{endpoint.replace("/", "_")}_100.json', 'w') as f:
-                json.dump(data, f)
-            process_cmc_data(data, '100')
-        except (ConnectionError, Timeout, TooManyRedirects) as e:
-            logging.error(f"Error while fetching data from {target}: {e}")
-
-def process_cmc_data(data, stop):
-    df = pd.DataFrame(data["data"])[["name", "symbol", "circulating_supply", "total_supply", "quote"]]
-    quote_df = pd.json_normalize(df['quote'].apply(lambda x: x['USD']))[["price", "percent_change_24h", "percent_change_7d", "percent_change_90d", "market_cap", "fully_diluted_market_cap", "last_updated"]]
-    df = df.drop("quote", axis=1)
-    df["percent_tokens_circulation"] = np.round((df["circulating_supply"] / df["total_supply"]) * 100, 1)
-    df = df.join(quote_df)
-    df["last_updated"] = pd.to_datetime(df["last_updated"])
-    save_cmc_data(df, stop)
-
-def save_cmc_data(df, stop):
-    output_file = f"output/top_{stop}_update.csv"
-    if os.path.isfile(output_file):
-        existing_data = pd.read_csv(output_file)
-        updated_data = pd.concat([existing_data, df], axis=0, ignore_index=True)
-        updated_data.drop_duplicates(subset=["symbol", "last_updated"], inplace=True)
-        updated_data.to_csv(output_file, index=False)
-    else:
-        df.to_csv(output_file, index=False)
-    logging.info("CMC data script execution completed.")
-
-# ---------------------- Execution ----------------------
-if __name__ == "__main__":
-    fetch_and_update_etherscan()
-    fetch_and_process_cmc_data()

utils/scrap_influencers_balance.py

@@ -0,0 +1,39 @@
+# utils/scrap_influencers_balance.py
+import requests
+import pandas as pd
+import json
+import os
+from dotenv import load_dotenv
+
+# Load environment variables
+load_dotenv()
+API_KEY = os.getenv('API_KEY_ETHERSCAN')
+URL_ETHERSCAN = "https://api.etherscan.io/api"
+
+def fetch_ether_balance(addresses):
+    params = {
+        'module': 'account',
+        'action': 'balancemulti',
+        'address': ','.join(addresses),
+        'tag': 'latest',
+        'apikey': API_KEY
+    }
+    response = requests.get(URL_ETHERSCAN, params=params)
+    return response.json()
+
+def main():
+    # Load influencers
+    with open("ressources/dict_influencers_addr.json", "r") as file:
+        influencers = json.load(file)
+
+    addresses = list(influencers.values())
+    response = fetch_ether_balance(addresses)
+
+    if response["status"] == "1":
+        df_balances = pd.DataFrame(response["result"])
+        df_balances.to_csv("output/influencers_balances.csv", index=False)
+    else:
+        print("Failed to fetch balances:", response.get("message", "Unknown error"))
+
+if __name__ == "__main__":
+    main()