graph done

app.py

@@ -10,6 +10,7 @@ from dotenv import load_dotenv
 from requests.exceptions import ConnectionError, Timeout, TooManyRedirects
 import plotly.express as px 
 import json
+import networkx as nx 
 
 # ------------------------ Environment Variables --------------------------
 
@@ -125,53 +126,205 @@ def display_greed_fear_index():
     except FileNotFoundError:
         st.error("Greed and Fear index data not available. Please wait for the next update cycle.")
 
-#-------------------------------------scheduler ----------------------------------
-
-# Function to execute the scraping functions
-def execute_etherscan_scraping():
-    subprocess.call(["python", "utils/scrap_etherscan.py"])
-    logging.info("Etherscan scraping completed")
-    threading.Timer(3600, execute_etherscan_scraping).start()
+def load_token_balances():
+    try:
+        return pd.read_csv("output/influencers_token_balances.csv")
+    except FileNotFoundError:
+        logging.warning("Token balances file not found.")
+        return pd.DataFrame()  # Return an empty DataFrame if file is not found
     
-# Balancer scrapping
-def execute_influencers_scraping():
-    subprocess.call(["python", "utils/scrap_influencers_balance.py"])
-    logging.info("Influencers balance scraping completed")
-    threading.Timer(3600, execute_influencers_scraping).start()  # Run every hour, for example
+def create_token_balance_bar_plot(df):
+    if df.empty:
+        return go.Figure()  # Return an empty figure if there is no data
 
-    
-# Function to execute the scraping functions
-def execute_cmc_scraping():
-    subprocess.call(["python", "utils/scrap_cmc.py"])
-    logging.info("CMC scraping completed")
-    threading.Timer(2592000 / 9000, execute_cmc_scraping).start()
+    fig = px.bar(df, x="Influencer", y="Balance", color="Token", barmode="group")
+    fig.update_layout(
+        title="Token Balances of Influencers",
+        xaxis_title="Influencer",
+        yaxis_title="Token Balance",
+        legend_title="Token"
+    )
+    return fig
 
+def get_top_buyers(df, token, top_n=5):
+    # Filter for selected token
+    token_df = df[df['tokenSymbol'] == token]
 
-# Function to execute the global metrics scraping
-def execute_global_metrics_scraping():
-    subprocess.call(["python", "utils/scrap_cmc_global_metrics.py"])
-    logging.info("Global metrics scraping completed")
-    threading.Timer(2592000 / 9000, execute_influencers_scraping).start()  # Run every hour, for example
+    # Assuming 'value' column holds the amount bought and 'from' column holds the buyer's address
+    top_buyers = token_df.groupby('from')['value'].sum().sort_values(ascending=False).head(top_n)
 
-def execute_greed_fear_index_scraping():
-    subprocess.call(["python", "utils/scrap_greed_fear_index.py"])
-    logging.info("Greed and Fear index scraping completed")
-    threading.Timer(3600, execute_greed_fear_index_scraping).start()  # Adjust the interval as needed
+    return top_buyers.reset_index()
+
+def plot_top_buyers(df):
+    fig = px.bar(df, x='from', y='value', title=f'Top 5 Buyers of {selected_token}',orientation="h")
+    fig.update_layout(xaxis_title="Address", yaxis_title="Total Amount Bought")
+    return fig
+
+def load_influencer_interactions(influencer_name):
+    try:
+        # Load the influencer addresses dictionary
+        with open("ressources/dict_influencers_addr.json", "r") as file:
+            influencers = json.load(file)
+        
+        # Get the address of the specified influencer
+        influencer_address = influencers.get(influencer_name, None)
+        if influencer_address is None:
+            return pd.DataFrame(), None
+
+        file_path = f"output/interactions_{influencer_name}.csv"
+        df = pd.read_csv(file_path)
+
+        # Keep only the 'from', 'to', and 'value' columns and remove duplicates
+        df = df[['from', 'to', 'value']].drop_duplicates()
+        return df, influencer_address
+    except FileNotFoundError:
+        return pd.DataFrame(), None
+
+
+def create_network_graph(df, influencer_name, influencer_address):
+    G = nx.Graph()
+
+    # Consider bidirectional interactions
+    df_bi = pd.concat([df.rename(columns={'from': 'to', 'to': 'from'}), df])
+    interaction_counts = df_bi.groupby(['from', 'to']).size().reset_index(name='count')
+    top_interactions = interaction_counts.sort_values('count', ascending=False).head(20)
+
+    # Add edges and nodes to the graph
+    for _, row in top_interactions.iterrows():
+        G.add_edge(row['from'], row['to'], weight=row['count'])
+        G.add_node(row['from'], type='sender')
+        G.add_node(row['to'], type='receiver')
+
+    # Node positions
+    pos = nx.spring_layout(G, weight='weight')
+
+    # Edge trace
+    edge_x = []
+    edge_y = []
+    edge_hover = []
+    for edge in G.edges(data=True):
+        x0, y0 = pos[edge[0]]
+        x1, y1 = pos[edge[1]]
+        edge_x.extend([x0, x1, None])
+        edge_y.extend([y0, y1, None])
+        edge_hover.append(f'Interactions: {edge[2]["weight"]}')
+
+    edge_trace = go.Scatter(
+        x=edge_x, y=edge_y,
+        line=dict(width=2, color='#888'),
+        hoverinfo='text',
+        text=edge_hover,
+        mode='lines')
+
+    # Node trace
+    node_x = []
+    node_y = []
+    node_hover = []
+    node_size = []
+
+    for node in G.nodes():
+        x, y = pos[node]
+        node_x.append(x)
+        node_y.append(y)
+        connections = len(G.edges(node))
+        interaction_sum = interaction_counts[interaction_counts['from'].eq(node) | interaction_counts['to'].eq(node)]['count'].sum()
+        node_hover_info = f'Address: {node}<br># of connections: {connections}<br># of interactions: {interaction_sum}'
+        if node == influencer_address:
+            node_hover_info = f'Influencer: {influencer_name}<br>' + node_hover_info
+            node_size.append(30)  # Central node size
+        else:
+            node_size.append(20)  # Other nodes size
+        node_hover.append(node_hover_info)
+
+    node_trace = go.Scatter(
+        x=node_x, y=node_y,
+        mode='markers',
+        hoverinfo='text',
+        text=node_hover,
+        marker=dict(
+            showscale=False,
+            color='blue',
+            size=node_size,
+            line=dict(width=2, color='black')))
+
+    # Create figure
+    fig = go.Figure(data=[edge_trace, node_trace],
+                    layout=go.Layout(
+                        title=f'<br>Network graph of wallet interactions for {influencer_name}',
+                        titlefont=dict(size=16),
+                        showlegend=False,
+                        hovermode='closest',
+                        margin=dict(b=20, l=5, r=5, t=40),
+                        xaxis=dict(showgrid=False, zeroline=False, showticklabels=False),
+                        yaxis=dict(showgrid=False, zeroline=False, showticklabels=False)))
+
+    return fig, top_interactions
 
 
-if "initialized" not in st.session_state:
-    # Start the scraping threads
-    threading.Thread(target=execute_etherscan_scraping).start()
-    threading.Thread(target=execute_cmc_scraping).start()
-    threading.Thread(target=execute_influencers_scraping).start()
-    threading.Thread(target=execute_global_metrics_scraping).start()
-    threading.Thread(target=execute_greed_fear_index_scraping).start()
-    st.session_state["initialized"] = True
+#-------------------------------------scheduler ----------------------------------
+
+# # Function to execute the scraping functions
+# def execute_etherscan_scraping():
+#     subprocess.call(["python", "utils/scrap_etherscan.py"])
+#     logging.info("Etherscan scraping completed")
+#     threading.Timer(3600, execute_etherscan_scraping).start()
+    
+# # Balancer scrapping
+# def execute_influencers_scraping():
+#     subprocess.call(["python", "utils/scrap_influencers_balance.py"])
+#     logging.info("Influencers balance scraping completed")
+#     threading.Timer(3600, execute_influencers_scraping).start() 
+
+    
+# # Function to execute the scraping functions
+# def execute_cmc_scraping():
+#     subprocess.call(["python", "utils/scrap_cmc.py"])
+#     logging.info("CMC scraping completed")
+#     threading.Timer(2592000 / 9000, execute_cmc_scraping).start()
+
+
+# # Function to execute the global metrics scraping
+# def execute_global_metrics_scraping():
+#     subprocess.call(["python", "utils/scrap_cmc_global_metrics.py"])
+#     logging.info("Global metrics scraping completed")
+#     threading.Timer(2592000 / 9000, execute_influencers_scraping).start() 
+
+# def execute_greed_fear_index_scraping():
+#     subprocess.call(["python", "utils/scrap_greed_fear_index.py"])
+#     logging.info("Greed and Fear index scraping completed")
+#     threading.Timer(3600, execute_greed_fear_index_scraping).start()  
+    
+# def execute_token_balances_scraping():
+#     subprocess.call(["python", "utils/extract_tokens_balances.py"])
+#     logging.info("Token balances scraping completed")
+#     threading.Timer(3600, execute_token_balances_scraping).start()  
+
+
+# if "initialized" not in st.session_state:
+#     # Start the scraping threads
+#     threading.Thread(target=execute_etherscan_scraping).start()
+#     threading.Thread(target=execute_cmc_scraping).start()
+#     threading.Thread(target=execute_influencers_scraping).start()
+#     threading.Thread(target=execute_global_metrics_scraping).start()
+#     threading.Thread(target=execute_greed_fear_index_scraping).start()
+#     threading.Thread(target=execute_token_balances_scraping).start()
+#     st.session_state["initialized"] = True
 
 #-------------------------------------streamlit ----------------------------------
 
 # Set the title and other page configurations
 st.title('Crypto Analysis')
+st.write("Welcome to the Crypto Analysis app. Please note that data is not updated automatically due to API plan limitations.")
+# Update Data Button
+if st.button("Scrap new data"):
+    # Execute the scripts in the 'utils' folder to update data
+    subprocess.call(["python", "utils/scrap_etherscan.py"])
+    subprocess.call(["python", "utils/scrap_cmc.py"])
+    subprocess.call(["python", "utils/scrap_influencers_balance.py"])
+    subprocess.call(["python", "utils/scrap_cmc_global_metrics.py"])
+    subprocess.call(["python", "utils/scrap_greed_fear_index.py"])
+    subprocess.call(["python", "utils/extract_tokens_balances.py"])
+    st.success("Data updated successfully!")
 
 st.header("Global Cryptocurrency Market Metrics")
 # Create two columns for the two plots
@@ -259,7 +412,7 @@ with col1:
     selected_token = st.selectbox('Select Token', df_etherscan['tokenSymbol'].unique(), index=0)
     # Filter the data based on the selected token
     filtered_df = df_etherscan[df_etherscan['tokenSymbol'] == selected_token]
-    # Plot the token value over time
+    # Plot the token volume over time
     st.plotly_chart(
         go.Figure(
             data=[
@@ -267,13 +420,13 @@ with col1:
                     x=filtered_df['timeStamp'],
                     y=filtered_df['value'],
                     mode='lines',
-                    name='Value over time'
+                    name='Volume over time'
                 )
             ],
             layout=go.Layout(
-                title='Token Value Over Time',
+                title='Token Volume Over Time',
                 yaxis=dict(
-                    title=f'Value ({selected_token})',
+                    title=f'Volume ({selected_token})',
                 ),
                 showlegend=True,
                 legend=go.layout.Legend(x=0, y=1.0),
@@ -284,7 +437,34 @@ with col1:
             )
         )
     )
+with col2:
+    # Token selection
+    selected_token = st.selectbox('Select a Token', df_etherscan['tokenSymbol'].unique())
+
+    # Processing data
+    top_buyers_df = get_top_buyers(df_etherscan, selected_token)
+
+    # Plotting
+    if not top_buyers_df.empty:
+        top_buyers_fig = plot_top_buyers(top_buyers_df)
+        top_buyers_fig.update_layout(
+            autosize=False,
+            width=300,
+            height=300)
+        st.plotly_chart(top_buyers_fig)
+    else:
+        st.write(f"No buying data available for {selected_token}")
 
+
+st.header("Influencers' Token Balances")
+token_balances_df = load_token_balances()
+col1, col2 = st.columns(2)
+with col1:
+    if not token_balances_df.empty:
+        token_balance_fig = create_token_balance_bar_plot(token_balances_df)
+        st.plotly_chart(token_balance_fig)
+    else:
+        st.write("No token balance data available.")
 with col2:
     influencers = load_influencers()
     with st.container():
@@ -342,7 +522,21 @@ with col2:
     else:
         logging.info("DataFrame is empty")
 
-
-st.header("Deep Dive into Specific Coins")
-
+# In the Streamlit app
+st.header("Wallet Interactions Network Graph")
+selected_influencer = st.selectbox("Select an Influencer", list(influencers.keys()))
+# Load interactions data for the selected influencer
+interactions_df, influencer_address = load_influencer_interactions(selected_influencer)
+if not interactions_df.empty:
+    # Generate the network graph and the table of top interactions
+    network_fig, top_interactions = create_network_graph(interactions_df, selected_influencer, influencer_address)
+    # Display the network graph
+    st.plotly_chart(network_fig)
+    # Display the table of top interactions
+    st.subheader(f"Top Interactions for {selected_influencer}")
+    st.table(top_interactions)
+else:
+    st.write(f"No wallet interaction data available for {selected_influencer}.")
+    
 #-------------------------------------end ----------------------------------
+

logs/scrapping/cmc_global_metrics.log

@@ -37,3 +37,27 @@
 2023-11-28 01:58:45,913 [INFO] - Global metrics data fetched successfully.
 2023-11-28 01:59:14,099 [INFO] - Global metrics data fetched successfully.
 2023-11-28 01:59:31,047 [INFO] - Global metrics data fetched successfully.
+2023-11-28 10:33:32,014 [INFO] - Global metrics data fetched successfully.
+2023-11-28 10:33:53,067 [INFO] - Global metrics data fetched successfully.
+2023-11-28 10:36:53,158 [INFO] - Global metrics data fetched successfully.
+2023-11-28 10:44:58,663 [INFO] - Global metrics data fetched successfully.
+2023-11-28 10:47:21,116 [INFO] - Global metrics data fetched successfully.
+2023-11-28 11:07:40,154 [INFO] - Global metrics data fetched successfully.
+2023-11-28 11:09:39,424 [ERROR] - Error fetching global metrics data: HTTPSConnectionPool(host='pro-api.coinmarketcap.com', port=443): Max retries exceeded with url: /v1/global-metrics/quotes/latest (Caused by NameResolutionError("<urllib3.connection.HTTPSConnection object at 0x7fb111a4e310>: Failed to resolve 'pro-api.coinmarketcap.com' ([Errno -3] Temporary failure in name resolution)"))
+2023-11-28 11:11:46,917 [ERROR] - Error fetching global metrics data: HTTPSConnectionPool(host='pro-api.coinmarketcap.com', port=443): Max retries exceeded with url: /v1/global-metrics/quotes/latest (Caused by NameResolutionError("<urllib3.connection.HTTPSConnection object at 0x7f02646b77d0>: Failed to resolve 'pro-api.coinmarketcap.com' ([Errno -3] Temporary failure in name resolution)"))
+2023-11-28 11:13:43,073 [ERROR] - Error fetching global metrics data: HTTPSConnectionPool(host='pro-api.coinmarketcap.com', port=443): Max retries exceeded with url: /v1/global-metrics/quotes/latest (Caused by NameResolutionError("<urllib3.connection.HTTPSConnection object at 0x7f757a99e350>: Failed to resolve 'pro-api.coinmarketcap.com' ([Errno -3] Temporary failure in name resolution)"))
+2023-11-28 11:31:55,662 [INFO] - Global metrics data fetched successfully.
+2023-11-28 11:33:16,437 [INFO] - Global metrics data fetched successfully.
+2023-11-28 11:38:31,535 [INFO] - Global metrics data fetched successfully.
+2023-11-28 11:41:56,440 [INFO] - Global metrics data fetched successfully.
+2023-11-28 11:42:08,580 [INFO] - Global metrics data fetched successfully.
+2023-11-28 11:43:43,358 [INFO] - Global metrics data fetched successfully.
+2023-11-28 11:57:15,728 [INFO] - Global metrics data fetched successfully.
+2023-11-28 11:58:10,309 [INFO] - Global metrics data fetched successfully.
+2023-11-28 12:05:40,716 [INFO] - Global metrics data fetched successfully.
+2023-11-28 12:06:18,520 [INFO] - Global metrics data fetched successfully.
+2023-11-28 12:08:03,238 [INFO] - Global metrics data fetched successfully.
+2023-11-28 12:18:14,152 [INFO] - Global metrics data fetched successfully.
+2023-11-28 13:10:39,099 [INFO] - Global metrics data fetched successfully.
+2023-11-28 13:12:08,480 [INFO] - Global metrics data fetched successfully.
+2023-11-28 13:22:26,139 [INFO] - Global metrics data fetched successfully.

logs/scrapping/cmc_scrapping.log

@@ -450,3 +450,107 @@
 2023-11-28 01:59:31,035 [INFO] - Function fetch_and_process_cmc_data executed in 0.22 seconds
 2023-11-28 01:59:54,297 [INFO] - CMC data script execution completed.
 2023-11-28 01:59:54,298 [INFO] - Function fetch_and_process_cmc_data executed in 0.24 seconds
+2023-11-28 10:33:32,105 [INFO] - CMC data script execution completed.
+2023-11-28 10:33:32,106 [INFO] - Function fetch_and_process_cmc_data executed in 0.24 seconds
+2023-11-28 10:33:53,199 [INFO] - CMC data script execution completed.
+2023-11-28 10:33:53,200 [INFO] - Function fetch_and_process_cmc_data executed in 0.31 seconds
+2023-11-28 10:36:53,233 [INFO] - CMC data script execution completed.
+2023-11-28 10:36:53,234 [INFO] - Function fetch_and_process_cmc_data executed in 0.24 seconds
+2023-11-28 10:38:20,700 [INFO] - CMC data script execution completed.
+2023-11-28 10:38:20,701 [INFO] - Function fetch_and_process_cmc_data executed in 0.25 seconds
+2023-11-28 10:38:41,968 [INFO] - CMC data script execution completed.
+2023-11-28 10:38:41,969 [INFO] - Function fetch_and_process_cmc_data executed in 0.51 seconds
+2023-11-28 10:41:41,771 [INFO] - CMC data script execution completed.
+2023-11-28 10:41:41,772 [INFO] - Function fetch_and_process_cmc_data executed in 0.23 seconds
+2023-11-28 10:43:09,822 [INFO] - CMC data script execution completed.
+2023-11-28 10:43:09,824 [INFO] - Function fetch_and_process_cmc_data executed in 0.80 seconds
+2023-11-28 10:43:30,480 [INFO] - CMC data script execution completed.
+2023-11-28 10:43:30,482 [INFO] - Function fetch_and_process_cmc_data executed in 0.25 seconds
+2023-11-28 10:44:58,815 [INFO] - CMC data script execution completed.
+2023-11-28 10:44:58,816 [INFO] - Function fetch_and_process_cmc_data executed in 0.32 seconds
+2023-11-28 10:47:21,447 [INFO] - CMC data script execution completed.
+2023-11-28 10:47:21,449 [INFO] - Function fetch_and_process_cmc_data executed in 0.49 seconds
+2023-11-28 10:52:10,034 [INFO] - CMC data script execution completed.
+2023-11-28 10:52:10,035 [INFO] - Function fetch_and_process_cmc_data executed in 0.26 seconds
+2023-11-28 10:56:58,591 [INFO] - CMC data script execution completed.
+2023-11-28 10:56:58,592 [INFO] - Function fetch_and_process_cmc_data executed in 0.24 seconds
+2023-11-28 11:01:47,106 [INFO] - CMC data script execution completed.
+2023-11-28 11:01:47,107 [INFO] - Function fetch_and_process_cmc_data executed in 0.25 seconds
+2023-11-28 11:06:35,611 [INFO] - CMC data script execution completed.
+2023-11-28 11:06:35,612 [INFO] - Function fetch_and_process_cmc_data executed in 0.24 seconds
+2023-11-28 11:07:40,264 [INFO] - CMC data script execution completed.
+2023-11-28 11:07:40,266 [INFO] - Function fetch_and_process_cmc_data executed in 0.29 seconds
+2023-11-28 11:09:39,426 [ERROR] - Error while fetching data from https://pro-api.coinmarketcap.com/v1/cryptocurrency/listings/latest: HTTPSConnectionPool(host='pro-api.coinmarketcap.com', port=443): Max retries exceeded with url: /v1/cryptocurrency/listings/latest?start=1&limit=100&convert=USD (Caused by NameResolutionError("<urllib3.connection.HTTPSConnection object at 0x7fc49cbd3650>: Failed to resolve 'pro-api.coinmarketcap.com' ([Errno -3] Temporary failure in name resolution)"))
+2023-11-28 11:09:39,426 [INFO] - Function fetch_and_process_cmc_data executed in 10.01 seconds
+2023-11-28 11:11:46,897 [ERROR] - Error while fetching data from https://pro-api.coinmarketcap.com/v1/cryptocurrency/listings/latest: HTTPSConnectionPool(host='pro-api.coinmarketcap.com', port=443): Max retries exceeded with url: /v1/cryptocurrency/listings/latest?start=1&limit=100&convert=USD (Caused by NameResolutionError("<urllib3.connection.HTTPSConnection object at 0x7f90ef296250>: Failed to resolve 'pro-api.coinmarketcap.com' ([Errno -3] Temporary failure in name resolution)"))
+2023-11-28 11:11:46,897 [INFO] - Function fetch_and_process_cmc_data executed in 10.01 seconds
+2023-11-28 11:13:43,052 [ERROR] - Error while fetching data from https://pro-api.coinmarketcap.com/v1/cryptocurrency/listings/latest: HTTPSConnectionPool(host='pro-api.coinmarketcap.com', port=443): Max retries exceeded with url: /v1/cryptocurrency/listings/latest?start=1&limit=100&convert=USD (Caused by NameResolutionError("<urllib3.connection.HTTPSConnection object at 0x7f07f5f9fb90>: Failed to resolve 'pro-api.coinmarketcap.com' ([Errno -3] Temporary failure in name resolution)"))
+2023-11-28 11:13:43,052 [INFO] - Function fetch_and_process_cmc_data executed in 10.01 seconds
+2023-11-28 11:14:37,758 [ERROR] - Error while fetching data from https://pro-api.coinmarketcap.com/v1/cryptocurrency/listings/latest: HTTPSConnectionPool(host='pro-api.coinmarketcap.com', port=443): Max retries exceeded with url: /v1/cryptocurrency/listings/latest?start=1&limit=100&convert=USD (Caused by NameResolutionError("<urllib3.connection.HTTPSConnection object at 0x7fdb32cd6a50>: Failed to resolve 'pro-api.coinmarketcap.com' ([Errno -3] Temporary failure in name resolution)"))
+2023-11-28 11:14:37,758 [INFO] - Function fetch_and_process_cmc_data executed in 10.01 seconds
+2023-11-28 11:16:35,556 [INFO] - CMC data script execution completed.
+2023-11-28 11:16:35,557 [INFO] - Function fetch_and_process_cmc_data executed in 0.31 seconds
+2023-11-28 11:18:31,730 [INFO] - CMC data script execution completed.
+2023-11-28 11:18:31,731 [INFO] - Function fetch_and_process_cmc_data executed in 0.31 seconds
+2023-11-28 11:19:26,248 [INFO] - CMC data script execution completed.
+2023-11-28 11:19:26,249 [INFO] - Function fetch_and_process_cmc_data executed in 0.24 seconds
+2023-11-28 11:21:24,060 [INFO] - CMC data script execution completed.
+2023-11-28 11:21:24,061 [INFO] - Function fetch_and_process_cmc_data executed in 0.24 seconds
+2023-11-28 11:23:20,350 [INFO] - CMC data script execution completed.
+2023-11-28 11:23:20,352 [INFO] - Function fetch_and_process_cmc_data executed in 0.33 seconds
+2023-11-28 11:24:14,814 [INFO] - CMC data script execution completed.
+2023-11-28 11:24:14,815 [INFO] - Function fetch_and_process_cmc_data executed in 0.27 seconds
+2023-11-28 11:26:12,552 [INFO] - CMC data script execution completed.
+2023-11-28 11:26:12,553 [INFO] - Function fetch_and_process_cmc_data executed in 0.24 seconds
+2023-11-28 11:28:08,878 [INFO] - CMC data script execution completed.
+2023-11-28 11:28:08,879 [INFO] - Function fetch_and_process_cmc_data executed in 0.25 seconds
+2023-11-28 11:29:03,395 [INFO] - CMC data script execution completed.
+2023-11-28 11:29:03,396 [INFO] - Function fetch_and_process_cmc_data executed in 0.33 seconds
+2023-11-28 11:31:01,093 [INFO] - CMC data script execution completed.
+2023-11-28 11:31:01,094 [INFO] - Function fetch_and_process_cmc_data executed in 0.25 seconds
+2023-11-28 11:31:55,758 [INFO] - CMC data script execution completed.
+2023-11-28 11:31:55,760 [INFO] - Function fetch_and_process_cmc_data executed in 0.27 seconds
+2023-11-28 11:33:16,503 [INFO] - CMC data script execution completed.
+2023-11-28 11:33:16,505 [INFO] - Function fetch_and_process_cmc_data executed in 0.25 seconds
+2023-11-28 11:38:31,661 [INFO] - CMC data script execution completed.
+2023-11-28 11:38:31,662 [INFO] - Function fetch_and_process_cmc_data executed in 0.25 seconds
+2023-11-28 11:41:56,517 [INFO] - CMC data script execution completed.
+2023-11-28 11:41:56,518 [INFO] - Function fetch_and_process_cmc_data executed in 0.25 seconds
+2023-11-28 11:42:08,632 [INFO] - CMC data script execution completed.
+2023-11-28 11:42:08,633 [INFO] - Function fetch_and_process_cmc_data executed in 0.25 seconds
+2023-11-28 11:43:43,468 [INFO] - CMC data script execution completed.
+2023-11-28 11:43:43,470 [INFO] - Function fetch_and_process_cmc_data executed in 0.27 seconds
+2023-11-28 11:48:32,034 [INFO] - CMC data script execution completed.
+2023-11-28 11:48:32,035 [INFO] - Function fetch_and_process_cmc_data executed in 0.25 seconds
+2023-11-28 11:53:20,611 [INFO] - CMC data script execution completed.
+2023-11-28 11:53:20,612 [INFO] - Function fetch_and_process_cmc_data executed in 0.33 seconds
+2023-11-28 11:57:15,865 [INFO] - CMC data script execution completed.
+2023-11-28 11:57:15,866 [INFO] - Function fetch_and_process_cmc_data executed in 0.24 seconds
+2023-11-28 11:58:10,401 [INFO] - CMC data script execution completed.
+2023-11-28 11:58:10,403 [INFO] - Function fetch_and_process_cmc_data executed in 0.24 seconds
+2023-11-28 12:05:41,082 [INFO] - CMC data script execution completed.
+2023-11-28 12:05:41,083 [INFO] - Function fetch_and_process_cmc_data executed in 0.54 seconds
+2023-11-28 12:06:18,449 [INFO] - CMC data script execution completed.
+2023-11-28 12:06:18,450 [INFO] - Function fetch_and_process_cmc_data executed in 0.25 seconds
+2023-11-28 12:08:03,421 [INFO] - CMC data script execution completed.
+2023-11-28 12:08:03,422 [INFO] - Function fetch_and_process_cmc_data executed in 0.35 seconds
+2023-11-28 12:11:07,111 [INFO] - CMC data script execution completed.
+2023-11-28 12:11:07,112 [INFO] - Function fetch_and_process_cmc_data executed in 0.25 seconds
+2023-11-28 12:12:51,988 [INFO] - CMC data script execution completed.
+2023-11-28 12:12:51,989 [INFO] - Function fetch_and_process_cmc_data executed in 0.25 seconds
+2023-11-28 12:15:55,603 [INFO] - CMC data script execution completed.
+2023-11-28 12:15:55,604 [INFO] - Function fetch_and_process_cmc_data executed in 0.25 seconds
+2023-11-28 12:17:40,504 [INFO] - CMC data script execution completed.
+2023-11-28 12:17:40,505 [INFO] - Function fetch_and_process_cmc_data executed in 0.26 seconds
+2023-11-28 12:18:14,263 [INFO] - CMC data script execution completed.
+2023-11-28 12:18:14,264 [INFO] - Function fetch_and_process_cmc_data executed in 0.24 seconds
+2023-11-28 12:23:02,901 [INFO] - CMC data script execution completed.
+2023-11-28 12:23:02,902 [INFO] - Function fetch_and_process_cmc_data executed in 0.33 seconds
+2023-11-28 13:10:39,209 [INFO] - CMC data script execution completed.
+2023-11-28 13:10:39,210 [INFO] - Function fetch_and_process_cmc_data executed in 0.28 seconds
+2023-11-28 13:12:08,803 [INFO] - CMC data script execution completed.
+2023-11-28 13:12:08,804 [INFO] - Function fetch_and_process_cmc_data executed in 0.44 seconds
+2023-11-28 13:16:57,350 [INFO] - CMC data script execution completed.
+2023-11-28 13:16:57,351 [INFO] - Function fetch_and_process_cmc_data executed in 0.27 seconds
+2023-11-28 13:22:25,021 [INFO] - CMC data script execution completed.
+2023-11-28 13:22:25,022 [INFO] - Function fetch_and_process_cmc_data executed in 0.26 seconds

logs/scrapping/etherscan_scrap.log

@@ -343,3 +343,65 @@
 2023-11-28 01:59:52,774 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
 2023-11-28 01:59:52,774 [INFO] - Etherscan scraping script execution completed.
 2023-11-28 01:59:52,774 [INFO] - Function fetch_and_update_etherscan executed in 21.97 seconds
+2023-11-28 10:33:33,310 [ERROR] - Invalid data format for transactions: Max rate limit reached
+2023-11-28 10:33:33,310 [ERROR] - 'timeStamp' key not found in the response data.
+2023-11-28 10:33:54,267 [ERROR] - Invalid data format for transactions: Max rate limit reached
+2023-11-28 10:33:54,268 [ERROR] - 'timeStamp' key not found in the response data.
+2023-11-28 10:37:14,138 [INFO] - Created files: 
+2023-11-28 10:37:14,138 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 10:37:14,138 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 10:37:14,138 [INFO] - Function fetch_and_update_etherscan executed in 21.15 seconds
+2023-11-28 10:44:59,904 [ERROR] - Invalid data format for transactions: Max rate limit reached
+2023-11-28 10:44:59,904 [ERROR] - 'timeStamp' key not found in the response data.
+2023-11-28 10:47:12,849 [INFO] - Created files: output/transactions_COPI.csv
+2023-11-28 10:47:12,849 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 10:47:12,849 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 10:47:12,849 [INFO] - Function fetch_and_update_etherscan executed in 22.67 seconds
+2023-11-28 10:47:25,889 [ERROR] - Invalid data format for transactions: Max rate limit reached
+2023-11-28 10:47:25,889 [ERROR] - 'timeStamp' key not found in the response data.
+2023-11-28 11:07:41,378 [ERROR] - Invalid data format for transactions: Max rate limit reached
+2023-11-28 11:07:41,378 [ERROR] - 'timeStamp' key not found in the response data.
+2023-11-28 11:31:56,906 [ERROR] - Invalid data format for transactions: Max rate limit reached
+2023-11-28 11:31:56,906 [ERROR] - 'timeStamp' key not found in the response data.
+2023-11-28 11:33:17,681 [ERROR] - Invalid data format for transactions: Max rate limit reached
+2023-11-28 11:33:17,682 [ERROR] - 'timeStamp' key not found in the response data.
+2023-11-28 11:38:55,405 [INFO] - Created files: 
+2023-11-28 11:38:55,405 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv, output/transactions_COPI.csv
+2023-11-28 11:38:55,405 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 11:38:55,405 [INFO] - Function fetch_and_update_etherscan executed in 24.05 seconds
+2023-11-28 11:41:57,687 [ERROR] - Invalid data format for transactions: Max rate limit reached
+2023-11-28 11:41:57,687 [ERROR] - 'timeStamp' key not found in the response data.
+2023-11-28 11:44:09,146 [INFO] - Created files: 
+2023-11-28 11:44:09,146 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv, output/transactions_COPI.csv
+2023-11-28 11:44:09,146 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 11:44:09,146 [INFO] - Function fetch_and_update_etherscan executed in 25.95 seconds
+2023-11-28 11:57:37,072 [INFO] - Created files: 
+2023-11-28 11:57:37,072 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv, output/transactions_COPI.csv
+2023-11-28 11:57:37,072 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 11:57:37,072 [INFO] - Function fetch_and_update_etherscan executed in 21.48 seconds
+2023-11-28 11:58:41,148 [INFO] - Created files: 
+2023-11-28 11:58:41,148 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv, output/transactions_COPI.csv
+2023-11-28 11:58:41,148 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 11:58:41,148 [INFO] - Function fetch_and_update_etherscan executed in 31.03 seconds
+2023-11-28 12:06:03,296 [INFO] - Created files: 
+2023-11-28 12:06:03,297 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv, output/transactions_COPI.csv
+2023-11-28 12:06:03,297 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 12:06:03,297 [INFO] - Function fetch_and_update_etherscan executed in 22.72 seconds
+2023-11-28 12:06:46,032 [INFO] - Created files: 
+2023-11-28 12:06:46,032 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv, output/transactions_COPI.csv
+2023-11-28 12:06:46,033 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 12:06:46,033 [INFO] - Function fetch_and_update_etherscan executed in 27.85 seconds
+2023-11-28 12:08:04,502 [ERROR] - Invalid data format for transactions: Max rate limit reached
+2023-11-28 12:08:04,502 [ERROR] - 'timeStamp' key not found in the response data.
+2023-11-28 12:18:34,929 [INFO] - Created files: 
+2023-11-28 12:18:34,929 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv, output/transactions_COPI.csv
+2023-11-28 12:18:34,929 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 12:18:34,929 [INFO] - Function fetch_and_update_etherscan executed in 20.92 seconds
+2023-11-28 13:10:40,360 [ERROR] - Invalid data format for transactions: Max rate limit reached
+2023-11-28 13:10:40,361 [ERROR] - 'timeStamp' key not found in the response data.
+2023-11-28 13:12:10,042 [ERROR] - Invalid data format for transactions: Max rate limit reached
+2023-11-28 13:12:10,042 [ERROR] - 'timeStamp' key not found in the response data.
+2023-11-28 13:22:24,524 [INFO] - Created files: 
+2023-11-28 13:22:24,524 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv, output/transactions_COPI.csv
+2023-11-28 13:22:24,524 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 13:22:24,524 [INFO] - Function fetch_and_update_etherscan executed in 20.44 seconds

logs/scrapping/greed_fear_index.log

@@ -21,3 +21,24 @@
 2023-11-28 01:58:46,004 [INFO] - Greed and Fear index data fetched successfully.
 2023-11-28 01:59:11,591 [INFO] - Greed and Fear index data fetched successfully.
 2023-11-28 01:59:31,101 [INFO] - Greed and Fear index data fetched successfully.
+2023-11-28 10:33:32,146 [INFO] - Greed and Fear index data fetched successfully.
+2023-11-28 10:33:53,293 [INFO] - Greed and Fear index data fetched successfully.
+2023-11-28 10:36:53,240 [INFO] - Greed and Fear index data fetched successfully.
+2023-11-28 10:44:58,843 [INFO] - Greed and Fear index data fetched successfully.
+2023-11-28 10:47:21,495 [INFO] - Greed and Fear index data fetched successfully.
+2023-11-28 11:07:40,243 [INFO] - Greed and Fear index data fetched successfully.
+2023-11-28 11:31:55,778 [INFO] - Greed and Fear index data fetched successfully.
+2023-11-28 11:33:16,553 [INFO] - Greed and Fear index data fetched successfully.
+2023-11-28 11:38:31,948 [INFO] - Greed and Fear index data fetched successfully.
+2023-11-28 11:41:56,556 [INFO] - Greed and Fear index data fetched successfully.
+2023-11-28 11:42:08,720 [INFO] - Greed and Fear index data fetched successfully.
+2023-11-28 11:43:43,570 [INFO] - Greed and Fear index data fetched successfully.
+2023-11-28 11:57:15,928 [INFO] - Greed and Fear index data fetched successfully.
+2023-11-28 11:58:10,485 [INFO] - Greed and Fear index data fetched successfully.
+2023-11-28 12:05:40,844 [INFO] - Greed and Fear index data fetched successfully.
+2023-11-28 12:06:19,435 [INFO] - Greed and Fear index data fetched successfully.
+2023-11-28 12:08:03,445 [INFO] - Greed and Fear index data fetched successfully.
+2023-11-28 12:18:14,299 [INFO] - Greed and Fear index data fetched successfully.
+2023-11-28 13:10:39,386 [INFO] - Greed and Fear index data fetched successfully.
+2023-11-28 13:12:08,710 [INFO] - Greed and Fear index data fetched successfully.
+2023-11-28 13:22:26,672 [INFO] - Greed and Fear index data fetched successfully.

logs/streamlit/front.log

@@ -673,3 +673,267 @@
 2023-11-28 01:59:32,265 [INFO] - Etherscan scraping completed
 2023-11-28 01:59:52,816 [INFO] - Etherscan scraping completed
 2023-11-28 01:59:54,340 [INFO] - CMC scraping completed
+2023-11-28 10:33:31,304 [INFO] - Streamlit app has started
+2023-11-28 10:33:31,944 [INFO] - Balances uploaded, shape of dataframe is (4, 2)
+2023-11-28 10:33:32,092 [INFO] - Global metrics scraping completed
+2023-11-28 10:33:32,170 [INFO] - CMC scraping completed
+2023-11-28 10:33:32,223 [INFO] - Greed and Fear index scraping completed
+2023-11-28 10:33:32,419 [INFO] - Influencers balance scraping completed
+2023-11-28 10:33:33,368 [INFO] - Etherscan scraping completed
+2023-11-28 10:33:42,956 [INFO] - Token balances scraping completed
+2023-11-28 10:33:52,392 [INFO] - Streamlit app has started
+2023-11-28 10:33:52,595 [INFO] - Balances uploaded, shape of dataframe is (4, 2)
+2023-11-28 10:33:53,119 [INFO] - Global metrics scraping completed
+2023-11-28 10:33:53,251 [INFO] - CMC scraping completed
+2023-11-28 10:33:53,346 [INFO] - Greed and Fear index scraping completed
+2023-11-28 10:33:53,430 [INFO] - Influencers balance scraping completed
+2023-11-28 10:33:54,322 [INFO] - Etherscan scraping completed
+2023-11-28 10:34:04,113 [INFO] - Token balances scraping completed
+2023-11-28 10:36:52,468 [INFO] - Streamlit app has started
+2023-11-28 10:36:52,686 [INFO] - Balances uploaded, shape of dataframe is (4, 2)
+2023-11-28 10:36:53,215 [INFO] - Global metrics scraping completed
+2023-11-28 10:36:53,303 [INFO] - Greed and Fear index scraping completed
+2023-11-28 10:36:53,304 [INFO] - CMC scraping completed
+2023-11-28 10:36:53,509 [INFO] - Influencers balance scraping completed
+2023-11-28 10:37:04,672 [INFO] - Token balances scraping completed
+2023-11-28 10:37:14,173 [INFO] - Etherscan scraping completed
+2023-11-28 10:38:20,767 [INFO] - CMC scraping completed
+2023-11-28 10:38:20,928 [INFO] - Influencers balance scraping completed
+2023-11-28 10:38:41,875 [INFO] - Influencers balance scraping completed
+2023-11-28 10:38:42,009 [INFO] - CMC scraping completed
+2023-11-28 10:41:41,814 [INFO] - CMC scraping completed
+2023-11-28 10:41:42,015 [INFO] - Influencers balance scraping completed
+2023-11-28 10:43:09,865 [INFO] - CMC scraping completed
+2023-11-28 10:43:30,528 [INFO] - CMC scraping completed
+2023-11-28 10:44:57,972 [INFO] - Streamlit app has started
+2023-11-28 10:44:58,530 [INFO] - Balances uploaded, shape of dataframe is (4, 2)
+2023-11-28 10:44:58,723 [INFO] - Global metrics scraping completed
+2023-11-28 10:44:58,900 [INFO] - CMC scraping completed
+2023-11-28 10:44:58,937 [INFO] - Greed and Fear index scraping completed
+2023-11-28 10:44:59,025 [INFO] - Influencers balance scraping completed
+2023-11-28 10:44:59,991 [INFO] - Etherscan scraping completed
+2023-11-28 10:45:08,425 [INFO] - Token balances scraping completed
+2023-11-28 10:47:20,352 [INFO] - Streamlit app has started
+2023-11-28 10:47:20,977 [INFO] - Balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 10:47:21,197 [INFO] - Global metrics scraping completed
+2023-11-28 10:47:21,523 [INFO] - CMC scraping completed
+2023-11-28 10:47:21,531 [INFO] - Influencers balance scraping completed
+2023-11-28 10:47:21,568 [INFO] - Greed and Fear index scraping completed
+2023-11-28 10:47:25,969 [INFO] - Etherscan scraping completed
+2023-11-28 10:47:30,999 [INFO] - Token balances scraping completed
+2023-11-28 10:52:10,018 [INFO] - Influencers balance scraping completed
+2023-11-28 10:52:10,094 [INFO] - CMC scraping completed
+2023-11-28 10:56:58,654 [INFO] - CMC scraping completed
+2023-11-28 11:01:47,146 [INFO] - CMC scraping completed
+2023-11-28 11:06:35,649 [INFO] - CMC scraping completed
+2023-11-28 11:07:39,400 [INFO] - Streamlit app has started
+2023-11-28 11:07:39,642 [INFO] - Balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 11:07:40,213 [INFO] - Global metrics scraping completed
+2023-11-28 11:07:40,326 [INFO] - Greed and Fear index scraping completed
+2023-11-28 11:07:40,342 [INFO] - CMC scraping completed
+2023-11-28 11:07:40,477 [INFO] - Influencers balance scraping completed
+2023-11-28 11:07:41,454 [INFO] - Etherscan scraping completed
+2023-11-28 11:07:49,803 [INFO] - Token balances scraping completed
+2023-11-28 11:09:28,933 [INFO] - Streamlit app has started
+2023-11-28 11:09:29,503 [INFO] - Balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 11:09:39,494 [INFO] - Greed and Fear index scraping completed
+2023-11-28 11:09:39,505 [INFO] - CMC scraping completed
+2023-11-28 11:09:39,512 [INFO] - Global metrics scraping completed
+2023-11-28 11:09:39,518 [INFO] - Influencers balance scraping completed
+2023-11-28 11:09:39,518 [INFO] - Token balances scraping completed
+2023-11-28 11:09:39,542 [INFO] - Etherscan scraping completed
+2023-11-28 11:10:01,250 [INFO] - Streamlit app has started
+2023-11-28 11:10:01,450 [INFO] - Balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 11:10:03,070 [INFO] - Streamlit app has started
+2023-11-28 11:10:03,256 [INFO] - Balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 11:11:36,331 [INFO] - Streamlit app has started
+2023-11-28 11:11:36,680 [INFO] - Balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 11:11:46,974 [INFO] - Greed and Fear index scraping completed
+2023-11-28 11:11:46,977 [INFO] - CMC scraping completed
+2023-11-28 11:11:46,982 [INFO] - Token balances scraping completed
+2023-11-28 11:11:46,983 [INFO] - Influencers balance scraping completed
+2023-11-28 11:11:47,009 [INFO] - Global metrics scraping completed
+2023-11-28 11:11:47,017 [INFO] - Etherscan scraping completed
+2023-11-28 11:13:32,451 [INFO] - Streamlit app has started
+2023-11-28 11:13:32,783 [INFO] - Balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 11:13:43,185 [INFO] - Influencers balance scraping completed
+2023-11-28 11:13:43,188 [INFO] - Greed and Fear index scraping completed
+2023-11-28 11:13:43,194 [INFO] - CMC scraping completed
+2023-11-28 11:13:43,195 [INFO] - Token balances scraping completed
+2023-11-28 11:13:43,216 [INFO] - Global metrics scraping completed
+2023-11-28 11:13:43,226 [INFO] - Etherscan scraping completed
+2023-11-28 11:13:43,690 [INFO] - Streamlit app has started
+2023-11-28 11:13:43,879 [INFO] - Balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 11:13:45,536 [INFO] - Streamlit app has started
+2023-11-28 11:13:45,748 [INFO] - Balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 11:14:37,809 [INFO] - CMC scraping completed
+2023-11-28 11:14:37,815 [INFO] - Influencers balance scraping completed
+2023-11-28 11:16:35,607 [INFO] - CMC scraping completed
+2023-11-28 11:16:35,810 [INFO] - Influencers balance scraping completed
+2023-11-28 11:18:31,768 [INFO] - CMC scraping completed
+2023-11-28 11:18:31,967 [INFO] - Influencers balance scraping completed
+2023-11-28 11:19:26,289 [INFO] - CMC scraping completed
+2023-11-28 11:21:24,099 [INFO] - CMC scraping completed
+2023-11-28 11:23:20,427 [INFO] - CMC scraping completed
+2023-11-28 11:24:14,863 [INFO] - CMC scraping completed
+2023-11-28 11:26:12,605 [INFO] - CMC scraping completed
+2023-11-28 11:28:08,923 [INFO] - CMC scraping completed
+2023-11-28 11:29:03,434 [INFO] - CMC scraping completed
+2023-11-28 11:31:01,135 [INFO] - CMC scraping completed
+2023-11-28 11:31:54,938 [INFO] - Streamlit app has started
+2023-11-28 11:31:55,541 [INFO] - Balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 11:31:55,713 [INFO] - Global metrics scraping completed
+2023-11-28 11:31:55,859 [INFO] - CMC scraping completed
+2023-11-28 11:31:55,877 [INFO] - Greed and Fear index scraping completed
+2023-11-28 11:31:56,011 [INFO] - Influencers balance scraping completed
+2023-11-28 11:31:56,994 [INFO] - Etherscan scraping completed
+2023-11-28 11:32:05,297 [INFO] - Token balances scraping completed
+2023-11-28 11:33:15,762 [INFO] - Streamlit app has started
+2023-11-28 11:33:16,303 [INFO] - Balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 11:33:16,318 [WARNING] - Wallet interactions file not found.
+2023-11-28 11:33:16,492 [INFO] - Global metrics scraping completed
+2023-11-28 11:33:16,571 [INFO] - CMC scraping completed
+2023-11-28 11:33:16,616 [INFO] - Greed and Fear index scraping completed
+2023-11-28 11:33:16,782 [INFO] - Influencers balance scraping completed
+2023-11-28 11:33:17,726 [INFO] - Etherscan scraping completed
+2023-11-28 11:33:26,028 [INFO] - Token balances scraping completed
+2023-11-28 11:38:30,855 [INFO] - Streamlit app has started
+2023-11-28 11:38:31,485 [INFO] - Balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 11:38:31,506 [WARNING] - Wallet interactions file not found.
+2023-11-28 11:38:31,626 [INFO] - Global metrics scraping completed
+2023-11-28 11:38:31,756 [INFO] - CMC scraping completed
+2023-11-28 11:38:31,956 [INFO] - Influencers balance scraping completed
+2023-11-28 11:38:32,039 [INFO] - Greed and Fear index scraping completed
+2023-11-28 11:38:45,284 [INFO] - Token balances scraping completed
+2023-11-28 11:38:55,442 [INFO] - Etherscan scraping completed
+2023-11-28 11:41:55,758 [INFO] - Streamlit app has started
+2023-11-28 11:41:56,042 [INFO] - Balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 11:41:56,500 [INFO] - Global metrics scraping completed
+2023-11-28 11:41:56,585 [INFO] - CMC scraping completed
+2023-11-28 11:41:56,619 [INFO] - Greed and Fear index scraping completed
+2023-11-28 11:41:56,844 [INFO] - Influencers balance scraping completed
+2023-11-28 11:41:57,729 [INFO] - Etherscan scraping completed
+2023-11-28 11:42:04,997 [INFO] - Token balances scraping completed
+2023-11-28 11:42:07,900 [INFO] - Streamlit app has started
+2023-11-28 11:42:08,497 [INFO] - Balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 11:42:08,646 [INFO] - Global metrics scraping completed
+2023-11-28 11:42:08,703 [INFO] - CMC scraping completed
+2023-11-28 11:42:08,778 [INFO] - Greed and Fear index scraping completed
+2023-11-28 11:42:08,929 [INFO] - Influencers balance scraping completed
+2023-11-28 11:42:11,373 [INFO] - Streamlit app has started
+2023-11-28 11:42:11,538 [INFO] - Balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 11:42:14,771 [INFO] - Token balances scraping completed
+2023-11-28 11:42:14,775 [INFO] - Etherscan scraping completed
+2023-11-28 11:43:42,678 [INFO] - Streamlit app has started
+2023-11-28 11:43:43,257 [INFO] - Balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 11:43:43,449 [INFO] - Global metrics scraping completed
+2023-11-28 11:43:43,570 [INFO] - CMC scraping completed
+2023-11-28 11:43:43,659 [INFO] - Greed and Fear index scraping completed
+2023-11-28 11:43:43,768 [INFO] - Influencers balance scraping completed
+2023-11-28 11:43:57,879 [INFO] - Token balances scraping completed
+2023-11-28 11:44:09,187 [INFO] - Etherscan scraping completed
+2023-11-28 11:48:32,077 [INFO] - CMC scraping completed
+2023-11-28 11:48:32,201 [INFO] - Influencers balance scraping completed
+2023-11-28 11:53:20,650 [INFO] - CMC scraping completed
+2023-11-28 11:57:15,074 [INFO] - Streamlit app has started
+2023-11-28 11:57:15,658 [INFO] - Balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 11:57:15,800 [INFO] - Global metrics scraping completed
+2023-11-28 11:57:15,990 [INFO] - CMC scraping completed
+2023-11-28 11:57:16,016 [INFO] - Greed and Fear index scraping completed
+2023-11-28 11:57:16,111 [INFO] - Influencers balance scraping completed
+2023-11-28 11:57:33,870 [INFO] - Token balances scraping completed
+2023-11-28 11:57:37,125 [INFO] - Etherscan scraping completed
+2023-11-28 11:58:09,595 [INFO] - Streamlit app has started
+2023-11-28 11:58:10,203 [INFO] - Balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 11:58:10,368 [INFO] - Global metrics scraping completed
+2023-11-28 11:58:10,480 [INFO] - CMC scraping completed
+2023-11-28 11:58:10,560 [INFO] - Greed and Fear index scraping completed
+2023-11-28 11:58:10,701 [INFO] - Influencers balance scraping completed
+2023-11-28 11:58:26,071 [INFO] - Token balances scraping completed
+2023-11-28 11:58:41,198 [INFO] - Etherscan scraping completed
+2023-11-28 12:05:40,063 [INFO] - Streamlit app has started
+2023-11-28 12:05:40,618 [INFO] - Balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 12:05:40,781 [INFO] - Global metrics scraping completed
+2023-11-28 12:05:40,923 [INFO] - Greed and Fear index scraping completed
+2023-11-28 12:05:41,107 [INFO] - Influencers balance scraping completed
+2023-11-28 12:05:41,152 [INFO] - CMC scraping completed
+2023-11-28 12:05:50,285 [INFO] - Token balances scraping completed
+2023-11-28 12:06:03,333 [INFO] - Etherscan scraping completed
+2023-11-28 12:06:17,658 [INFO] - Streamlit app has started
+2023-11-28 12:06:18,235 [INFO] - Balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 12:06:18,587 [INFO] - CMC scraping completed
+2023-11-28 12:06:18,622 [INFO] - Global metrics scraping completed
+2023-11-28 12:06:18,773 [INFO] - Influencers balance scraping completed
+2023-11-28 12:06:19,470 [INFO] - Greed and Fear index scraping completed
+2023-11-28 12:06:28,837 [INFO] - Token balances scraping completed
+2023-11-28 12:06:46,069 [INFO] - Etherscan scraping completed
+2023-11-28 12:08:02,516 [INFO] - Streamlit app has started
+2023-11-28 12:08:02,791 [INFO] - Balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 12:08:03,294 [INFO] - Global metrics scraping completed
+2023-11-28 12:08:03,507 [INFO] - CMC scraping completed
+2023-11-28 12:08:03,549 [INFO] - Greed and Fear index scraping completed
+2023-11-28 12:08:03,614 [INFO] - Influencers balance scraping completed
+2023-11-28 12:08:04,554 [INFO] - Etherscan scraping completed
+2023-11-28 12:08:12,880 [INFO] - Token balances scraping completed
+2023-11-28 12:11:07,154 [INFO] - CMC scraping completed
+2023-11-28 12:11:07,427 [INFO] - Influencers balance scraping completed
+2023-11-28 12:12:52,039 [INFO] - Influencers balance scraping completed
+2023-11-28 12:12:52,039 [INFO] - CMC scraping completed
+2023-11-28 12:15:55,641 [INFO] - CMC scraping completed
+2023-11-28 12:17:40,543 [INFO] - CMC scraping completed
+2023-11-28 12:18:13,441 [INFO] - Streamlit app has started
+2023-11-28 12:18:14,059 [INFO] - Balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 12:18:14,249 [INFO] - Global metrics scraping completed
+2023-11-28 12:18:14,352 [INFO] - CMC scraping completed
+2023-11-28 12:18:14,396 [INFO] - Greed and Fear index scraping completed
+2023-11-28 12:18:14,527 [INFO] - Influencers balance scraping completed
+2023-11-28 12:18:25,177 [INFO] - Token balances scraping completed
+2023-11-28 12:18:34,978 [INFO] - Etherscan scraping completed
+2023-11-28 12:23:02,938 [INFO] - CMC scraping completed
+2023-11-28 12:23:03,024 [INFO] - Influencers balance scraping completed
+2023-11-28 13:10:38,358 [INFO] - Streamlit app has started
+2023-11-28 13:10:39,038 [INFO] - Balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 13:10:39,179 [INFO] - Global metrics scraping completed
+2023-11-28 13:10:39,286 [INFO] - CMC scraping completed
+2023-11-28 13:10:39,461 [INFO] - Greed and Fear index scraping completed
+2023-11-28 13:10:39,471 [INFO] - Influencers balance scraping completed
+2023-11-28 13:10:40,423 [INFO] - Etherscan scraping completed
+2023-11-28 13:10:49,824 [INFO] - Token balances scraping completed
+2023-11-28 13:12:07,786 [INFO] - Streamlit app has started
+2023-11-28 13:12:08,380 [INFO] - Balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 13:12:08,544 [INFO] - Global metrics scraping completed
+2023-11-28 13:12:08,770 [INFO] - Greed and Fear index scraping completed
+2023-11-28 13:12:08,867 [INFO] - CMC scraping completed
+2023-11-28 13:12:09,173 [INFO] - Influencers balance scraping completed
+2023-11-28 13:12:10,086 [INFO] - Etherscan scraping completed
+2023-11-28 13:12:19,423 [INFO] - Token balances scraping completed
+2023-11-28 13:16:57,314 [INFO] - Influencers balance scraping completed
+2023-11-28 13:16:57,399 [INFO] - CMC scraping completed
+2023-11-28 13:19:41,896 [INFO] - Streamlit app has started
+2023-11-28 13:19:42,325 [INFO] - Balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 13:19:49,733 [INFO] - Streamlit app has started
+2023-11-28 13:19:49,920 [INFO] - Balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 13:19:52,110 [INFO] - Streamlit app has started
+2023-11-28 13:19:52,289 [INFO] - Balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 13:20:35,829 [INFO] - Streamlit app has started
+2023-11-28 13:20:36,023 [INFO] - Balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 13:21:04,545 [INFO] - Streamlit app has started
+2023-11-28 13:21:04,734 [INFO] - Balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 13:21:13,159 [INFO] - Streamlit app has started
+2023-11-28 13:21:13,354 [INFO] - Balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 13:21:14,994 [INFO] - Streamlit app has started
+2023-11-28 13:21:15,190 [INFO] - Balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 13:21:17,132 [INFO] - Streamlit app has started
+2023-11-28 13:21:17,328 [INFO] - Balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 13:21:19,109 [INFO] - Streamlit app has started
+2023-11-28 13:21:19,294 [INFO] - Balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 13:21:21,201 [INFO] - Streamlit app has started
+2023-11-28 13:21:21,413 [INFO] - Balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 13:21:28,530 [INFO] - Streamlit app has started
+2023-11-28 13:21:28,720 [INFO] - Balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 13:21:32,344 [INFO] - Streamlit app has started
+2023-11-28 13:21:32,537 [INFO] - Balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 13:21:58,369 [INFO] - Streamlit app has started
+2023-11-28 13:21:58,557 [INFO] - Balances uploaded, shape of dataframe is (3, 2)
+2023-11-28 13:22:03,724 [INFO] - Streamlit app has started
+2023-11-28 13:22:36,893 [INFO] - Balances uploaded, shape of dataframe is (3, 2)

output/all_influencers_interactions.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56fa4ff0ebacf5eca419e06beb2a4c09444b4bc1285f9503271e4e157693514a
+size 6455502

output/cmc_data_v1_cryptocurrency_listings_latest_100.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:49f394b714b63bcb568bc1f85ea5a8fe308067977994fc77a11d7401f613b2ea
-size 119061
+oid sha256:fcb005b97a284701233e7b75b95cb1dd6edcc4d21608b0a77f60a708e4e03557
+size 118855

output/global_metrics.csv

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:56d354bb20eac786e584d13e27b0648dbfff97321da9cfea34ab2b5a937e956a
-size 2058
+oid sha256:f2ea68072d857f8df4fa012a3b3812958b6b429e244a7bb676ea3c8c3fd16fb8
+size 2062

output/greed_fear_index.csv

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4eee12b593c936615d9b103bf596fbcb999542a4a68987aff4100b9d5396cf15
+oid sha256:9dbd09283b7fecc48bc63ddae9f75a5ef7870736b506d1c9a98b90e1774fbb60
 size 363

output/influencers_balances.csv

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0a132619b2c97f10e5a84b873ea52f108b2a9e61d1f18f8d2aeaf4140a22b7bc
-size 247
+oid sha256:58780b31635e55bb240338631191ea8b067eab050a4c807a537d427079369202
+size 186

output/influencers_token_balances.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11885bd488cef73d210438ea5e0d733aa02aac1004e72657ea7fda0b08511cf5
+size 92

output/interactions_alexBecker.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3965f63ee83a15c803c501606a8406f2302e404728e60011c7851474fb6fb533
+size 3965237

output/interactions_elliotrades.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eab057a6c694ff9771c3e665a384d30ca8181c21adb8aea8c384b04bc2c51356
+size 5527711

output/interactions_jrny.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8228ad2cab966159a31e46eba2003847b59e83cfdf943e33198f185fab27cf97
+size 832545

output/top_100_update.csv

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c00ca112a17aa4596689a06be0226051c6ef155402a24d5f61d8b4562b3a1d8c
-size 1325534
+oid sha256:acce0c2d2784bfaa236f703cc90d437e4ee74b317ed09b101284a3a444d01827
+size 1967500

output/transactions_APE.csv

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:047ff00b9a1610e53ccf954fddcb4dff40cf5805f98090465f82072150590d1c
-size 3491358
+oid sha256:9342bac98f54aa2f8cd068e79138987b4f6973978458564e8eec9cc39b36ea41
+size 3761017

output/transactions_AXIE.csv

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c973cac0473cbbe02c7bdba3704a97a5819fb130118d9cd99b8b1667552b8a60
-size 1646172
+oid sha256:929bfd9aa07adaead0c1823711e6835df4708c1a3a397d484e029eb796b46a58
+size 1885537

output/transactions_COPI.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca886705a085bd9b737a8d97efdcaa3366c48b0dccd95a3aefc95a8a04009edb
+size 65583

output/transactions_GALA.csv

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b52b4143f26394c83aafdce92babe5a91d78e5f47ab824dfbe1e5428caa09be2
-size 4222917
+oid sha256:b6efe06c2df1429d02f9f774a74e31002991a378d1f5eac272ab22f147e30869
+size 4725457

output/transactions_MANA.csv

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:da79712ca9bbe25335e2b9c4ce28b9aa551a0a84ff013aa95b8dd22ec3f0bc12
-size 2254698
+oid sha256:f4b2e20bb735de1bb97298c715bee0ccffd5b6f593df6b98334a1fb30506a5e3
+size 2398224

output/transactions_PET.csv

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1ffde90e51dd3bedbe01ab04e38ea840fd7d9c01094c6f0b6d4ee3779ed3e09c
-size 885
+oid sha256:82630c63358642367e62796d2323691c7ca82a115cbeabc043774851d9f8644e
+size 1249

output/transactions_WEAOPON.csv

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:beac438541c99c972ae4322faae5e06a27171e15bb73436849aeed530e80b4da
-size 6082
+oid sha256:947b350533c3ce072db8988341ce5e1d6ae30c42c99f1da14492b4812ae4c2a4
+size 6453

requirements.txt

@@ -10,4 +10,7 @@ streamlit
 requests
 plotly
 nbformat
-schedule
+schedule
+networkx
+scipy
+matplotlib

ressources/dict_influencers_addr.json

@@ -1,6 +1,5 @@
 {
     "alexBecker": "0x4d224452801aced8b2f0aebe155379bb5d594381",
     "elliotrades": "0xc02f318365582557a482eb5d1834dfd7af4a3f59",
-    "jrny": "0x08c1ae7e46d4a13b766566033b5c47c735e19f6f",
-    "momo": "0x17770D4A8C0d068be7eB5b7AF453f9E176E52464"
+    "jrny": "0x08c1ae7e46d4a13b766566033b5c47c735e19f6f"
 }

ressources/dict_tokens_addr.json

@@ -3,5 +3,6 @@
 "GALA":"0xd1d2Eb1B1e90B638588728b4130137D262C87cae",
 "MANA":"0x0f5d2fb29fb7d3cfee444a200298f468908cc942",
 "PET":"0xb870679a7fa65b924026f496de7f27c1dd0e5c5f",
-"WEAOPON":"0x0b8057c3cc676c329c25c1d0cd27776efa73762d"
+"WEAOPON":"0x0b8057c3cc676c329c25c1d0cd27776efa73762d",
+"COPI":"0x42Baf1f659D765C65ADE5BB7E08eb2C680360d9d"
 }

utils/extract_tokens_balances.py

@@ -0,0 +1,50 @@
+import os
+import json
+import requests
+import pandas as pd
+from dotenv import load_dotenv
+
+# Load environment variables
+load_dotenv()
+ETHERSCAN_API_KEY = os.getenv("API_KEY_ETHERSCAN")
+TOKENS_TO_WATCH = "ressources/dict_tokens_addr.json"  # Path to the file with tokens to watch
+INFLUENCERS_FILE = "ressources/dict_influencers_addr.json"  # Path to the file with influencers' addresses
+
+def fetch_token_balance(address, token_contract):
+    url = f"https://api.etherscan.io/api?module=account&action=tokenbalance&contractaddress={token_contract}&address={address}&tag=latest&apikey={ETHERSCAN_API_KEY}"
+    response = requests.get(url)
+    if response.status_code == 200:
+        data = response.json()
+        if data["status"] == "1":
+            balance = int(data["result"])
+            return balance
+        else:
+            print(f"Error fetching token balance: {data.get('message')}")
+    else:
+        print(f"Error: HTTP {response.status_code}")
+    return 0
+
+def main():
+    # Load tokens and influencers
+    with open(TOKENS_TO_WATCH, "r") as file:
+        tokens_to_watch = json.load(file)
+    with open(INFLUENCERS_FILE, "r") as file:
+        influencers = json.load(file)
+
+    balances = []
+    for name, address in influencers.items():
+        for token_name, token_contract in tokens_to_watch.items():
+            balance = fetch_token_balance(address, token_contract)
+            if balance > 0:  # Optionally check if balance is above a certain threshold
+                balances.append({
+                    "Influencer": name,
+                    "Token": token_name,
+                    "Balance": balance / (10 ** 18)  # Convert from Wei
+                })
+
+    # Save to CSV
+    df = pd.DataFrame(balances)
+    df.to_csv("output/influencers_token_balances.csv", index=False)
+
+if __name__ == "__main__":
+    main()

utils/extract_wallet_interactions.py

@@ -0,0 +1,50 @@
+import requests
+import pandas as pd
+import json
+import os
+from dotenv import load_dotenv
+
+# Load environment variables
+load_dotenv()
+API_KEY = os.getenv('API_KEY_ETHERSCAN')
+URL_ETHERSCAN = os.getenv('URL_ETHERSCAN')
+
+def get_token_transactions(address, action):
+    params = {
+        'module': 'account',
+        'action': action,  # tokentx for ERC20, tokennfttx for ERC721, token1155tx for ERC1155
+        'address': address,
+        'startblock': 0,
+        'endblock': 99999999,
+        'sort': 'asc',
+        'apikey': API_KEY
+    }
+    response = requests.get(URL_ETHERSCAN, params=params)
+    if response.status_code == 200:
+        data = response.json()
+        return data['result']
+    else:
+        print(f"Error fetching transactions for address {address} with action {action}")
+        return []
+
+def combine_token_transactions(address):
+    erc20_transactions = get_token_transactions(address, 'tokentx')
+    erc721_transactions = get_token_transactions(address, 'tokennfttx')
+    erc1155_transactions = get_token_transactions(address, 'token1155tx')
+
+    combined_transactions = erc20_transactions + erc721_transactions + erc1155_transactions
+    return combined_transactions
+
+def save_transactions_to_csv(transactions, influencer_name):
+    df = pd.DataFrame(transactions)
+    df.to_csv(f"output/interactions_{influencer_name}.csv", index=False)
+
+if __name__ == "__main__":
+    # Load influencers
+    with open("ressources/dict_influencers_addr.json", "r") as file:
+        influencers = json.load(file)
+
+    # Extract and save interactions for all influencers
+    for name, address in influencers.items():
+        transactions = combine_token_transactions(address)
+        save_transactions_to_csv(transactions, name)