more graphs

.env

@@ -9,6 +9,10 @@ API_KEY_CMC=8057498e-ad35-465c-8359-8f6cc9d1ae1b
 URL_ETHERSCAN=https://api.etherscan.io/api
 API_KEY_ETHERSCAN=VVJFE7IG5WSK8P2UJIJN5MST5UBQEZYVYW
 
+# RapidAPI
+RAPIDAPI_KEY=0478c28d44msh9c841a123ff12d1p1da763jsn4b51c433d736
+RapidAPI-Host=fear-and-greed-index.p.rapidapi.com/v1/fgi
+
 # Logging
 LOG_FOLDER=./logs/scrapping/
 LOG_STREAMLIT=./logs/streamlit/

app.py

@@ -50,28 +50,122 @@ for filename in os.listdir('output'):
 df_cmc = safe_read_csv("output/top_100_update.csv", sep=',')
 df_cmc = df_cmc[df_cmc["last_updated"] == df_cmc["last_updated"].max()]
 
+# Global metrics about the market
+def load_global_metrics():
+    try:
+        return pd.read_csv("output/global_metrics.csv")
+    except FileNotFoundError:
+        logging.warning("Global metrics file not found.")
+        return pd.DataFrame()  # Return an empty DataFrame if file is not found
+
+# Load influencers
+def load_influencers():
+    try:
+        with open("ressources/dict_influencers_addr.json", "r") as file:
+            return json.load(file)
+    except Exception as e:
+        st.error(f"Error loading influencers: {e}")
+        return {}
+
+
+def create_dominance_pie_chart(df_global_metrics):
+    # Extract BTC and ETH dominance
+    btc_dominance = df_global_metrics['btc_dominance'].iloc[0]
+    eth_dominance = df_global_metrics['eth_dominance'].iloc[0]
+    # Calculate the dominance of other cryptocurrencies
+    others_dominance = 100 - btc_dominance - eth_dominance
+    #print(btc_dominance,eth_dominance,others_dominance)
+    # Prepare data for pie chart
+    dominance_data = {
+        'Cryptocurrency': ['BTC', 'ETH', 'Others'],
+        'Dominance': [btc_dominance, eth_dominance, others_dominance]
+    }
+    df_dominance = pd.DataFrame(dominance_data)
+    # Create a pie chart
+    fig = px.pie(df_dominance, values='Dominance', names='Cryptocurrency', title='Market Cap Dominance')
+    return fig
+
+def display_greed_fear_index():
+    try:
+        df = pd.read_csv('output/greed_fear_index.csv')
+
+        # Prepare data for plotting
+        time_periods = ['One Year Ago', 'One Month Ago', 'One Week Ago', 'Previous Close', 'Now']
+        values = [
+            df['fgi_oneYearAgo_value'].iloc[0],
+            df['fgi_oneMonthAgo_value'].iloc[0],
+            df['fgi_oneWeekAgo_value'].iloc[0],
+            df['fgi_previousClose_value'].iloc[0],
+            df['fgi_now_value'].iloc[0]
+        ]
+        labels = [
+            df['fgi_oneYearAgo_valueText'].iloc[0],
+            df['fgi_oneMonthAgo_valueText'].iloc[0],
+            df['fgi_oneWeekAgo_valueText'].iloc[0],
+            df['fgi_previousClose_valueText'].iloc[0],
+            df['fgi_now_valueText'].iloc[0]
+        ]
+
+        # Create a Plotly figure
+        fig = go.Figure(data=[
+            go.Scatter(x=time_periods, y=values, mode='lines+markers+text', text=labels, textposition='top center')
+        ])
+
+        # Update layout
+        fig.update_layout(
+            title='Fear and Greed Index Over Time',
+            xaxis_title='Time Period',
+            yaxis_title='Index Value',
+            yaxis=dict(range=[0, 100])  # Fear and Greed index ranges from 0 to 100
+        )
+
+        # Display the figure
+        st.plotly_chart(fig)
+
+    except FileNotFoundError:
+        st.error("Greed and Fear index data not available. Please wait for the next update cycle.")
+
+#-------------------------------------scheduler ----------------------------------
+
 # Function to execute the scraping functions
 def execute_etherscan_scraping():
     subprocess.call(["python", "utils/scrap_etherscan.py"])
     logging.info("Etherscan scraping completed")
     threading.Timer(3600, execute_etherscan_scraping).start()
     
+# Balancer scrapping
+def execute_influencers_scraping():
+    subprocess.call(["python", "utils/scrap_influencers_balance.py"])
+    logging.info("Influencers balance scraping completed")
+    threading.Timer(3600, execute_influencers_scraping).start()  # Run every hour, for example
+
+    
 # Function to execute the scraping functions
 def execute_cmc_scraping():
     subprocess.call(["python", "utils/scrap_cmc.py"])
     logging.info("CMC scraping completed")
     threading.Timer(2592000 / 9000, execute_cmc_scraping).start()
 
-def execute_influencers_scraping():
-    subprocess.call(["python", "utils/scrap_influencers_balance.py"])
-    logging.info("Influencers balance scraping completed")
-    threading.Timer(3600, execute_influencers_scraping).start()  # Run every hour, for example
+
+# Function to execute the global metrics scraping
+def execute_global_metrics_scraping():
+    subprocess.call(["python", "utils/scrap_cmc_global_metrics.py"])
+    logging.info("Global metrics scraping completed")
+    threading.Timer(2592000 / 9000, execute_influencers_scraping).start()  # Run every hour, for example
+
+def execute_greed_fear_index_scraping():
+    subprocess.call(["python", "utils/scrap_greed_fear_index.py"])
+    logging.info("Greed and Fear index scraping completed")
+    threading.Timer(3600, execute_greed_fear_index_scraping).start()  # Adjust the interval as needed
+
 
 if "initialized" not in st.session_state:
     # Start the scraping threads
     threading.Thread(target=execute_etherscan_scraping).start()
     threading.Thread(target=execute_cmc_scraping).start()
     threading.Thread(target=execute_influencers_scraping).start()
+    threading.Thread(target=execute_global_metrics_scraping).start()
+    threading.Thread(target=execute_greed_fear_index_scraping).start()
     st.session_state["initialized"] = True
 
 #-------------------------------------streamlit ----------------------------------
@@ -79,176 +173,176 @@ if "initialized" not in st.session_state:
 # Set the title and other page configurations
 st.title('Crypto Analysis')
 
+st.header("Global Cryptocurrency Market Metrics")
 # Create two columns for the two plots
 col1, col2 = st.columns(2)
+global_metrics_df = load_global_metrics()
+display_greed_fear_index()
 
-with st.container():
-
-    with col1:
-        # etherscan
-        selected_token = st.selectbox('Select Token', df_etherscan['tokenSymbol'].unique(), index=0)
-        # Filter the data based on the selected token
-        filtered_df = df_etherscan[df_etherscan['tokenSymbol'] == selected_token]
-        # Plot the token value over time
-        st.plotly_chart(
-            go.Figure(
-                data=[
-                    go.Scatter(
-                        x=filtered_df['timeStamp'],
-                        y=filtered_df['value'],
-                        mode='lines',
-                        name='Value over time'
-                    )
-                ],
-                layout=go.Layout(
-                    title='Token Value Over Time',
-                    yaxis=dict(
-                        title=f'Value ({selected_token})',
-                    ),
-                    showlegend=True,
-                    legend=go.layout.Legend(x=0, y=1.0),
-                    margin=go.layout.Margin(l=40, r=0, t=40, b=30),
-                    width=500,
-                    height=500
-
-                )
-            )
+    
+st.write(global_metrics_df)
+with col1:
+    # Create and display the pie chart
+    dominance_fig = create_dominance_pie_chart(global_metrics_df)
+    dominance_fig.update_layout(
+        autosize=False,
+        width=300,
+        height=300,)
+    st.plotly_chart(dominance_fig)
+with col2:
+    # cmc
+    selected_var = st.selectbox('Select Var', ["percent_change_24h","percent_change_7d","percent_change_90d"], index=0)
+    # Sort the DataFrame by the 'percent_change_24h' column in ascending order
+    df_sorted = df_cmc.sort_values(by=selected_var, ascending=False)
+    # Select the top 10 and worst 10 rows
+    top_10 = df_sorted.head(10)
+    worst_10 = df_sorted.tail(10)
+    # Combine the top and worst dataframes for plotting
+    combined_df = pd.concat([top_10, worst_10], axis=0)
+    max_abs_val = max(abs(combined_df[selected_var].min()), abs(combined_df[selected_var].max()))
+
+    # Create a bar plot for the top 10 with a green color scale
+    fig = go.Figure(data=[
+        go.Bar(
+            x=top_10["symbol"],
+            y=top_10[selected_var],
+            marker_color='rgb(0,100,0)',  # Green color for top 10
+            hovertext= "Name : "+top_10["name"].astype(str)+ '<br>' +
+                    selected_var + " : " + top_10["percent_tokens_circulation"].astype(str) + '<br>' +
+                    'Market Cap: ' + top_10["market_cap"].astype(str) + '<br>' +
+                    'Fully Diluted Market Cap: ' + top_10["fully_diluted_market_cap"].astype(str) + '<br>' +
+                    'Last Updated: ' + top_10["last_updated"].astype(str),
+            name="top_10"
+        )
+    ])
+
+    # Add the worst 10 to the same plot with a red color scale
+    fig.add_traces(go.Bar(
+            x=worst_10["symbol"],
+            y=worst_10[selected_var],
+            marker_color='rgb(255,0,0)',  # Red color for worst 10
+            hovertext="Name:"+worst_10["name"].astype(str)+ '<br>' +
+                    selected_var + " : " + worst_10["percent_tokens_circulation"].astype(str) + '<br>' +
+                    'Market Cap: ' + worst_10["market_cap"].astype(str) + '<br>' +
+                    'Fully Diluted Market Cap: ' + worst_10["fully_diluted_market_cap"].astype(str) + '<br>' +
+                    'Last Updated: ' + worst_10["last_updated"].astype(str),
+            name="worst_10"
         )
+    )
+
+    # Customize aspect
+    fig.update_traces(marker_line_color='rgb(8,48,107)', marker_line_width=1.5, opacity=0.8)
+    fig.update_layout(title_text=f'Top 10 and Worst 10 by {selected_var.split("_")[-1]} Percentage Change')
+    fig.update_xaxes(categoryorder='total ascending')
+    fig.update_layout(
+        autosize=False,
+        width=300,
+        height=300,
+        margin=dict(
+            l=50,
+            r=50,
+            b=100,
+            t=100,
+            pad=4
+        ),
+        #paper_bgcolor="LightSteelBlue",
+    )
+    st.plotly_chart(fig)
+        
+    
 
-    with col2:
-        # cmc
-        selected_var = st.selectbox('Select Token', ["percent_change_24h","percent_change_7d","percent_change_90d"], index=0)
-        # Sort the DataFrame by the 'percent_change_24h' column in ascending order
-        df_sorted = df_cmc.sort_values(by=selected_var, ascending=False)
-        # Select the top 10 and worst 10 rows
-        top_10 = df_sorted.head(10)
-        worst_10 = df_sorted.tail(10)
-        # Combine the top and worst dataframes for plotting
-        combined_df = pd.concat([top_10, worst_10], axis=0)
-        max_abs_val = max(abs(combined_df[selected_var].min()), abs(combined_df[selected_var].max()))
-
-        # Create a bar plot for the top 10 with a green color scale
-        fig = go.Figure(data=[
-            go.Bar(
-                x=top_10["symbol"],
-                y=top_10[selected_var],
-                marker_color='rgb(0,100,0)',  # Green color for top 10
-                hovertext= "Name : "+top_10["name"].astype(str)+ '<br>' +
-                        selected_var + " : " + top_10["percent_tokens_circulation"].astype(str) + '<br>' +
-                        'Market Cap: ' + top_10["market_cap"].astype(str) + '<br>' +
-                        'Fully Diluted Market Cap: ' + top_10["fully_diluted_market_cap"].astype(str) + '<br>' +
-                        'Last Updated: ' + top_10["last_updated"].astype(str),
-                name="top_10"
-            )
-        ])
 
-        # Add the worst 10 to the same plot with a red color scale
-        fig.add_traces(go.Bar(
-                x=worst_10["symbol"],
-                y=worst_10[selected_var],
-                marker_color='rgb(255,0,0)',  # Red color for worst 10
-                hovertext="Name:"+worst_10["name"].astype(str)+ '<br>' +
-                        selected_var + " : " + worst_10["percent_tokens_circulation"].astype(str) + '<br>' +
-                        'Market Cap: ' + worst_10["market_cap"].astype(str) + '<br>' +
-                        'Fully Diluted Market Cap: ' + worst_10["fully_diluted_market_cap"].astype(str) + '<br>' +
-                        'Last Updated: ' + worst_10["last_updated"].astype(str),
-                name="worst_10"
+st.header("Deep Dive into Specific Coins")
+col1, col2 = st.columns(2)
+with col1:
+    # etherscan
+    selected_token = st.selectbox('Select Token', df_etherscan['tokenSymbol'].unique(), index=0)
+    # Filter the data based on the selected token
+    filtered_df = df_etherscan[df_etherscan['tokenSymbol'] == selected_token]
+    # Plot the token value over time
+    st.plotly_chart(
+        go.Figure(
+            data=[
+                go.Scatter(
+                    x=filtered_df['timeStamp'],
+                    y=filtered_df['value'],
+                    mode='lines',
+                    name='Value over time'
+                )
+            ],
+            layout=go.Layout(
+                title='Token Value Over Time',
+                yaxis=dict(
+                    title=f'Value ({selected_token})',
+                ),
+                showlegend=True,
+                legend=go.layout.Legend(x=0, y=1.0),
+                margin=go.layout.Margin(l=40, r=0, t=40, b=30),
+                width=300,
+                height=300,
+
             )
         )
+    )
 
-        # Customize aspect
-        fig.update_traces(marker_line_color='rgb(8,48,107)', marker_line_width=1.5, opacity=0.8)
-        fig.update_layout(title_text=f'Top 10 and Worst 10 by {selected_var.split("_")[-1]} Percentage Change')
-        fig.update_xaxes(categoryorder='total ascending')
+with col2:
+    influencers = load_influencers()
+    with st.container():
+        influencer_input = st.text_input("Follow this name:address", placeholder="e.g., alice:0x123...ABC")
+        if st.button("Add Influencer"):
+            if ":" in influencer_input:
+                try:
+                    new_influencer_name, new_influencer_addr = influencer_input.split(":")
+                    influencers[new_influencer_name.strip()] = new_influencer_addr.strip()
+                    with open("ressources/dict_influencers_addr.json", "w") as file:
+                        json.dump(influencers, file, indent=4)
+                    st.success(f"Influencer {new_influencer_name} added")
+                    subprocess.call(["python", "utils/scrap_influencers_balance.py"])
+                    st.success("Balance updated")
+                except ValueError:
+                    st.error("Invalid format. Please enter as 'name:address'")
+            else:
+                st.error("Please enter the influencer details as 'name:address'")
+    # Load Ether balances
+    try:
+        df_balances = pd.read_csv("output/influencers_balances.csv")
+        logging.info(f"Balances uploaded, shape of dataframe is {df_balances.shape}")
+        #st.write("DataFrame Loaded:", df_balances)  # Debugging line
+    except FileNotFoundError:
+        st.error("Balance data not found. Please wait for the next update cycle.")
+        df_balances = pd.DataFrame()
+        
+    # Inverting the influencers dictionary
+    inverted_influencers = {v.lower(): k for k, v in influencers.items()}
+
+    if not df_balances.empty:
+        df_balances["balance"] = df_balances["balance"].astype(float) / 1e18  # Convert Wei to Ether
+        df_balances = df_balances.rename(columns={"account": "address"})
+
+        # Ensure addresses are in the same format as in the inverted dictionary (e.g., lowercase)
+        df_balances["address"] = df_balances["address"].str.lower()
+
+        # Perform the mapping
+        df_balances["influencer"] = df_balances["address"].map(inverted_influencers)
+        #st.write("Mapped DataFrame:", df_balances)  # Debugging line
+
+        fig = px.bar(df_balances, y="influencer", x="balance",orientation="h")
+        fig.update_layout(
+            title='Ether Balances of Influencers',
+            xaxis=dict(
+                title='Balance in eth',
+                titlefont_size=16,
+                tickfont_size=14,
+            ))
         fig.update_layout(
             autosize=False,
-            width=500,
-            height=500,
-            margin=dict(
-                l=50,
-                r=50,
-                b=100,
-                t=100,
-                pad=4
-            ),
-            #paper_bgcolor="LightSteelBlue",
-        )
+            width=300,
+            height=400,)
         st.plotly_chart(fig)
-
-with st.container():
-    with col1:
-
-        # Load influencers
-        def load_influencers():
-            try:
-                with open("ressources/dict_influencers_addr.json", "r") as file:
-                    return json.load(file)
-            except Exception as e:
-                st.error(f"Error loading influencers: {e}")
-                return {}
-
-        influencers = load_influencers()
-
-        new_influencer_name = st.text_input("New Influencer Name")
-        new_influencer_addr = st.text_input("New Influencer Address")
-        if st.button("Add Influencer"):
-            if new_influencer_name and new_influencer_addr:
-                influencers[new_influencer_name] = new_influencer_addr
-                with open("ressources/dict_influencers_addr.json", "w") as file:
-                    json.dump(influencers, file, indent=4)
-                st.success(f"Influencer {new_influencer_name} added")
-                subprocess.call(["python", "utils/scrap_influencers_balance.py"])
-                st.success(f"Balance updated")
-            else:
-                st.error("Please enter both name and address")
-
-        # Load Ether balances
-        try:
-            df_balances = pd.read_csv("output/influencers_balances.csv")
-            logging.info(f"Balances uploaded, shape of dataframe is {df_balances.shape}")
-            #st.write("DataFrame Loaded:", df_balances)  # Debugging line
-        except FileNotFoundError:
-            st.error("Balance data not found. Please wait for the next update cycle.")
-            df_balances = pd.DataFrame()
-            
-        # Inverting the influencers dictionary
-        inverted_influencers = {v.lower(): k for k, v in influencers.items()}
-
-        if not df_balances.empty:
-            df_balances["balance"] = df_balances["balance"].astype(float) / 1e18  # Convert Wei to Ether
-            df_balances = df_balances.rename(columns={"account": "address"})
-
-            # Ensure addresses are in the same format as in the inverted dictionary (e.g., lowercase)
-            df_balances["address"] = df_balances["address"].str.lower()
-
-            # Perform the mapping
-            df_balances["influencer"] = df_balances["address"].map(inverted_influencers)
-            #st.write("Mapped DataFrame:", df_balances)  # Debugging line
-
-            fig = px.bar(df_balances, y="influencer", x="balance",orientation="h")
-            fig.update_layout(
-                title='Ether Balances of Influencers',
-                xaxis=dict(
-                    title='Balance in eth',
-                    titlefont_size=16,
-                    tickfont_size=14,
-                ))
-            fig.update_layout(
-                autosize=False,
-                width=500,
-                height=500,
-                margin=dict(
-                    l=50,
-                    r=50,
-                    b=100,
-                    t=100,
-                    pad=4
-                ))
-
-            st.plotly_chart(fig)
-        else:
-            logging.info("DataFrame is empty")
+    else:
+        logging.info("DataFrame is empty")
 
 
+st.header("Deep Dive into Specific Coins")
 
 #-------------------------------------end ----------------------------------

logs/scrapping/cmc_global_metrics.log

@@ -0,0 +1,39 @@
+2023-11-28 00:50:57,937 [INFO] - Global metrics data fetched successfully.
+2023-11-28 00:58:56,006 [INFO] - Global metrics data fetched successfully.
+2023-11-28 00:59:02,658 [INFO] - Global metrics data fetched successfully.
+2023-11-28 01:03:47,694 [INFO] - Global metrics data fetched successfully.
+2023-11-28 01:04:31,840 [INFO] - Global metrics data fetched successfully.
+2023-11-28 01:05:34,103 [INFO] - Global metrics data fetched successfully.
+2023-11-28 01:06:29,302 [INFO] - Global metrics data fetched successfully.
+2023-11-28 01:06:54,867 [INFO] - Global metrics data fetched successfully.
+2023-11-28 01:07:47,211 [INFO] - Global metrics data fetched successfully.
+2023-11-28 01:09:48,009 [INFO] - Global metrics data fetched successfully.
+2023-11-28 01:10:24,084 [INFO] - Global metrics data fetched successfully.
+2023-11-28 01:11:18,437 [INFO] - Global metrics data fetched successfully.
+2023-11-28 01:13:28,350 [INFO] - Global metrics data fetched successfully.
+2023-11-28 01:16:32,821 [INFO] - Global metrics data fetched successfully.
+2023-11-28 01:17:20,532 [INFO] - Global metrics data fetched successfully.
+2023-11-28 01:17:41,673 [INFO] - Global metrics data fetched successfully.
+2023-11-28 01:25:17,079 [INFO] - Global metrics data fetched successfully.
+2023-11-28 01:27:11,845 [INFO] - Global metrics data fetched successfully.
+2023-11-28 01:31:19,587 [INFO] - Global metrics data fetched successfully.
+2023-11-28 01:32:47,950 [INFO] - Global metrics data fetched successfully.
+2023-11-28 01:35:41,363 [INFO] - Global metrics data fetched successfully.
+2023-11-28 01:35:54,519 [INFO] - Global metrics data fetched successfully.
+2023-11-28 01:36:20,790 [INFO] - Global metrics data fetched successfully.
+2023-11-28 01:44:36,067 [INFO] - Global metrics data fetched successfully.
+2023-11-28 01:45:20,379 [INFO] - Global metrics data fetched successfully.
+2023-11-28 01:45:41,239 [INFO] - Global metrics data fetched successfully.
+2023-11-28 01:46:46,269 [INFO] - Global metrics data fetched successfully.
+2023-11-28 01:47:28,441 [INFO] - Global metrics data fetched successfully.
+2023-11-28 01:50:17,187 [INFO] - Global metrics data fetched successfully.
+2023-11-28 01:50:40,465 [INFO] - Global metrics data fetched successfully.
+2023-11-28 01:51:45,391 [INFO] - Global metrics data fetched successfully.
+2023-11-28 01:53:02,970 [INFO] - Global metrics data fetched successfully.
+2023-11-28 01:53:15,977 [INFO] - Global metrics data fetched successfully.
+2023-11-28 01:54:01,426 [INFO] - Global metrics data fetched successfully.
+2023-11-28 01:56:53,378 [INFO] - Global metrics data fetched successfully.
+2023-11-28 01:57:56,119 [INFO] - Global metrics data fetched successfully.
+2023-11-28 01:58:45,913 [INFO] - Global metrics data fetched successfully.
+2023-11-28 01:59:14,099 [INFO] - Global metrics data fetched successfully.
+2023-11-28 01:59:31,047 [INFO] - Global metrics data fetched successfully.

logs/scrapping/cmc_scrapping.log

@@ -302,3 +302,151 @@
 2023-11-28 00:48:02,132 [INFO] - Function fetch_and_process_cmc_data executed in 0.21 seconds
 2023-11-28 00:48:07,702 [INFO] - CMC data script execution completed.
 2023-11-28 00:48:07,703 [INFO] - Function fetch_and_process_cmc_data executed in 0.19 seconds
+2023-11-28 00:59:02,688 [INFO] - CMC data script execution completed.
+2023-11-28 00:59:02,689 [INFO] - Function fetch_and_process_cmc_data executed in 0.22 seconds
+2023-11-28 01:03:47,948 [INFO] - CMC data script execution completed.
+2023-11-28 01:03:47,949 [INFO] - Function fetch_and_process_cmc_data executed in 0.42 seconds
+2023-11-28 01:03:51,262 [INFO] - CMC data script execution completed.
+2023-11-28 01:03:51,264 [INFO] - Function fetch_and_process_cmc_data executed in 0.26 seconds
+2023-11-28 01:04:31,918 [INFO] - CMC data script execution completed.
+2023-11-28 01:04:31,919 [INFO] - Function fetch_and_process_cmc_data executed in 0.24 seconds
+2023-11-28 01:05:34,136 [INFO] - CMC data script execution completed.
+2023-11-28 01:05:34,138 [INFO] - Function fetch_and_process_cmc_data executed in 0.22 seconds
+2023-11-28 01:06:29,317 [INFO] - CMC data script execution completed.
+2023-11-28 01:06:29,318 [INFO] - Function fetch_and_process_cmc_data executed in 0.22 seconds
+2023-11-28 01:06:54,905 [INFO] - CMC data script execution completed.
+2023-11-28 01:06:54,906 [INFO] - Function fetch_and_process_cmc_data executed in 0.22 seconds
+2023-11-28 01:07:47,315 [INFO] - CMC data script execution completed.
+2023-11-28 01:07:47,316 [INFO] - Function fetch_and_process_cmc_data executed in 0.30 seconds
+2023-11-28 01:08:37,503 [INFO] - CMC data script execution completed.
+2023-11-28 01:08:37,504 [INFO] - Function fetch_and_process_cmc_data executed in 1.31 seconds
+2023-11-28 01:08:39,703 [INFO] - CMC data script execution completed.
+2023-11-28 01:08:39,704 [INFO] - Function fetch_and_process_cmc_data executed in 0.20 seconds
+2023-11-28 01:09:20,445 [INFO] - CMC data script execution completed.
+2023-11-28 01:09:20,445 [INFO] - Function fetch_and_process_cmc_data executed in 0.22 seconds
+2023-11-28 01:09:47,827 [INFO] - CMC data script execution completed.
+2023-11-28 01:09:47,827 [INFO] - Function fetch_and_process_cmc_data executed in 0.22 seconds
+2023-11-28 01:10:22,664 [INFO] - CMC data script execution completed.
+2023-11-28 01:10:22,665 [INFO] - Function fetch_and_process_cmc_data executed in 0.21 seconds
+2023-11-28 01:10:24,176 [INFO] - CMC data script execution completed.
+2023-11-28 01:10:24,177 [INFO] - Function fetch_and_process_cmc_data executed in 0.24 seconds
+2023-11-28 01:11:18,835 [INFO] - CMC data script execution completed.
+2023-11-28 01:11:18,836 [INFO] - Function fetch_and_process_cmc_data executed in 0.56 seconds
+2023-11-28 01:11:19,197 [INFO] - CMC data script execution completed.
+2023-11-28 01:11:19,198 [INFO] - Function fetch_and_process_cmc_data executed in 1.55 seconds
+2023-11-28 01:11:43,447 [INFO] - CMC data script execution completed.
+2023-11-28 01:11:43,448 [INFO] - Function fetch_and_process_cmc_data executed in 0.22 seconds
+2023-11-28 01:12:35,796 [INFO] - CMC data script execution completed.
+2023-11-28 01:12:35,797 [INFO] - Function fetch_and_process_cmc_data executed in 0.22 seconds
+2023-11-28 01:13:26,106 [INFO] - CMC data script execution completed.
+2023-11-28 01:13:26,107 [INFO] - Function fetch_and_process_cmc_data executed in 0.36 seconds
+2023-11-28 01:13:28,424 [INFO] - CMC data script execution completed.
+2023-11-28 01:13:28,425 [INFO] - Function fetch_and_process_cmc_data executed in 0.25 seconds
+2023-11-28 01:13:28,505 [INFO] - CMC data script execution completed.
+2023-11-28 01:13:28,506 [INFO] - Function fetch_and_process_cmc_data executed in 0.30 seconds
+2023-11-28 01:14:09,153 [INFO] - CMC data script execution completed.
+2023-11-28 01:14:09,154 [INFO] - Function fetch_and_process_cmc_data executed in 0.46 seconds
+2023-11-28 01:14:36,334 [INFO] - CMC data script execution completed.
+2023-11-28 01:14:36,335 [INFO] - Function fetch_and_process_cmc_data executed in 0.24 seconds
+2023-11-28 01:15:11,168 [INFO] - CMC data script execution completed.
+2023-11-28 01:15:11,169 [INFO] - Function fetch_and_process_cmc_data executed in 0.23 seconds
+2023-11-28 01:15:12,645 [INFO] - CMC data script execution completed.
+2023-11-28 01:15:12,646 [INFO] - Function fetch_and_process_cmc_data executed in 0.19 seconds
+2023-11-28 01:16:07,331 [INFO] - CMC data script execution completed.
+2023-11-28 01:16:07,332 [INFO] - Function fetch_and_process_cmc_data executed in 0.25 seconds
+2023-11-28 01:16:07,665 [INFO] - CMC data script execution completed.
+2023-11-28 01:16:07,666 [INFO] - Function fetch_and_process_cmc_data executed in 0.19 seconds
+2023-11-28 01:16:32,906 [INFO] - CMC data script execution completed.
+2023-11-28 01:16:32,907 [INFO] - Function fetch_and_process_cmc_data executed in 0.23 seconds
+2023-11-28 01:17:20,642 [INFO] - CMC data script execution completed.
+2023-11-28 01:17:20,643 [INFO] - Function fetch_and_process_cmc_data executed in 0.30 seconds
+2023-11-28 01:17:42,199 [INFO] - CMC data script execution completed.
+2023-11-28 01:17:42,200 [INFO] - Function fetch_and_process_cmc_data executed in 0.72 seconds
+2023-11-28 01:21:21,478 [INFO] - CMC data script execution completed.
+2023-11-28 01:21:21,480 [INFO] - Function fetch_and_process_cmc_data executed in 0.22 seconds
+2023-11-28 01:22:10,409 [INFO] - CMC data script execution completed.
+2023-11-28 01:22:10,410 [INFO] - Function fetch_and_process_cmc_data executed in 1.46 seconds
+2023-11-28 01:22:30,690 [INFO] - CMC data script execution completed.
+2023-11-28 01:22:30,692 [INFO] - Function fetch_and_process_cmc_data executed in 0.23 seconds
+2023-11-28 01:25:17,142 [INFO] - CMC data script execution completed.
+2023-11-28 01:25:17,144 [INFO] - Function fetch_and_process_cmc_data executed in 0.23 seconds
+2023-11-28 01:26:09,961 [INFO] - CMC data script execution completed.
+2023-11-28 01:26:09,962 [INFO] - Function fetch_and_process_cmc_data executed in 0.23 seconds
+2023-11-28 01:26:58,890 [INFO] - CMC data script execution completed.
+2023-11-28 01:26:58,891 [INFO] - Function fetch_and_process_cmc_data executed in 0.23 seconds
+2023-11-28 01:27:11,915 [INFO] - CMC data script execution completed.
+2023-11-28 01:27:11,916 [INFO] - Function fetch_and_process_cmc_data executed in 0.24 seconds
+2023-11-28 01:27:19,209 [INFO] - CMC data script execution completed.
+2023-11-28 01:27:19,210 [INFO] - Function fetch_and_process_cmc_data executed in 0.23 seconds
+2023-11-28 01:30:05,781 [INFO] - CMC data script execution completed.
+2023-11-28 01:30:05,782 [INFO] - Function fetch_and_process_cmc_data executed in 0.32 seconds
+2023-11-28 01:30:58,460 [INFO] - CMC data script execution completed.
+2023-11-28 01:30:58,461 [INFO] - Function fetch_and_process_cmc_data executed in 0.21 seconds
+2023-11-28 01:31:19,627 [INFO] - CMC data script execution completed.
+2023-11-28 01:31:19,628 [INFO] - Function fetch_and_process_cmc_data executed in 0.22 seconds
+2023-11-28 01:32:47,998 [INFO] - CMC data script execution completed.
+2023-11-28 01:32:47,999 [INFO] - Function fetch_and_process_cmc_data executed in 0.25 seconds
+2023-11-28 01:35:41,433 [INFO] - CMC data script execution completed.
+2023-11-28 01:35:41,434 [INFO] - Function fetch_and_process_cmc_data executed in 0.24 seconds
+2023-11-28 01:35:54,793 [INFO] - CMC data script execution completed.
+2023-11-28 01:35:54,794 [INFO] - Function fetch_and_process_cmc_data executed in 0.47 seconds
+2023-11-28 01:36:20,948 [INFO] - CMC data script execution completed.
+2023-11-28 01:36:20,949 [INFO] - Function fetch_and_process_cmc_data executed in 0.32 seconds
+2023-11-28 01:40:43,280 [INFO] - CMC data script execution completed.
+2023-11-28 01:40:43,281 [INFO] - Function fetch_and_process_cmc_data executed in 0.22 seconds
+2023-11-28 01:41:09,498 [INFO] - CMC data script execution completed.
+2023-11-28 01:41:09,499 [INFO] - Function fetch_and_process_cmc_data executed in 0.24 seconds
+2023-11-28 01:44:36,132 [INFO] - CMC data script execution completed.
+2023-11-28 01:44:36,134 [INFO] - Function fetch_and_process_cmc_data executed in 0.24 seconds
+2023-11-28 01:45:20,464 [INFO] - CMC data script execution completed.
+2023-11-28 01:45:20,465 [INFO] - Function fetch_and_process_cmc_data executed in 0.22 seconds
+2023-11-28 01:45:31,764 [INFO] - CMC data script execution completed.
+2023-11-28 01:45:31,765 [INFO] - Function fetch_and_process_cmc_data executed in 0.23 seconds
+2023-11-28 01:45:41,395 [INFO] - CMC data script execution completed.
+2023-11-28 01:45:41,396 [INFO] - Function fetch_and_process_cmc_data executed in 0.27 seconds
+2023-11-28 01:45:58,226 [INFO] - CMC data script execution completed.
+2023-11-28 01:45:58,227 [INFO] - Function fetch_and_process_cmc_data executed in 0.48 seconds
+2023-11-28 01:46:46,340 [INFO] - CMC data script execution completed.
+2023-11-28 01:46:46,341 [INFO] - Function fetch_and_process_cmc_data executed in 0.22 seconds
+2023-11-28 01:47:28,500 [INFO] - CMC data script execution completed.
+2023-11-28 01:47:28,502 [INFO] - Function fetch_and_process_cmc_data executed in 0.23 seconds
+2023-11-28 01:50:17,261 [INFO] - CMC data script execution completed.
+2023-11-28 01:50:17,262 [INFO] - Function fetch_and_process_cmc_data executed in 0.25 seconds
+2023-11-28 01:50:40,550 [INFO] - CMC data script execution completed.
+2023-11-28 01:50:40,552 [INFO] - Function fetch_and_process_cmc_data executed in 0.25 seconds
+2023-11-28 01:51:45,479 [INFO] - CMC data script execution completed.
+2023-11-28 01:51:45,480 [INFO] - Function fetch_and_process_cmc_data executed in 0.25 seconds
+2023-11-28 01:52:17,048 [INFO] - CMC data script execution completed.
+2023-11-28 01:52:17,049 [INFO] - Function fetch_and_process_cmc_data executed in 0.23 seconds
+2023-11-28 01:53:03,014 [INFO] - CMC data script execution completed.
+2023-11-28 01:53:03,015 [INFO] - Function fetch_and_process_cmc_data executed in 0.22 seconds
+2023-11-28 01:53:16,039 [INFO] - CMC data script execution completed.
+2023-11-28 01:53:16,040 [INFO] - Function fetch_and_process_cmc_data executed in 0.23 seconds
+2023-11-28 01:54:01,326 [INFO] - CMC data script execution completed.
+2023-11-28 01:54:01,328 [INFO] - Function fetch_and_process_cmc_data executed in 0.23 seconds
+2023-11-28 01:55:05,780 [INFO] - CMC data script execution completed.
+2023-11-28 01:55:05,781 [INFO] - Function fetch_and_process_cmc_data executed in 0.23 seconds
+2023-11-28 01:55:29,145 [INFO] - CMC data script execution completed.
+2023-11-28 01:55:29,146 [INFO] - Function fetch_and_process_cmc_data executed in 0.31 seconds
+2023-11-28 01:56:34,067 [INFO] - CMC data script execution completed.
+2023-11-28 01:56:34,068 [INFO] - Function fetch_and_process_cmc_data executed in 0.23 seconds
+2023-11-28 01:56:53,461 [INFO] - CMC data script execution completed.
+2023-11-28 01:56:53,461 [INFO] - Function fetch_and_process_cmc_data executed in 0.23 seconds
+2023-11-28 01:57:05,520 [INFO] - CMC data script execution completed.
+2023-11-28 01:57:05,521 [INFO] - Function fetch_and_process_cmc_data executed in 0.23 seconds
+2023-11-28 01:57:51,560 [INFO] - CMC data script execution completed.
+2023-11-28 01:57:51,561 [INFO] - Function fetch_and_process_cmc_data executed in 0.23 seconds
+2023-11-28 01:57:56,188 [INFO] - CMC data script execution completed.
+2023-11-28 01:57:56,189 [INFO] - Function fetch_and_process_cmc_data executed in 0.24 seconds
+2023-11-28 01:58:04,605 [INFO] - CMC data script execution completed.
+2023-11-28 01:58:04,606 [INFO] - Function fetch_and_process_cmc_data executed in 0.25 seconds
+2023-11-28 01:58:46,237 [INFO] - CMC data script execution completed.
+2023-11-28 01:58:46,238 [INFO] - Function fetch_and_process_cmc_data executed in 0.49 seconds
+2023-11-28 01:58:49,804 [INFO] - CMC data script execution completed.
+2023-11-28 01:58:49,804 [INFO] - Function fetch_and_process_cmc_data executed in 0.21 seconds
+2023-11-28 01:59:11,481 [INFO] - CMC data script execution completed.
+2023-11-28 01:59:11,482 [INFO] - Function fetch_and_process_cmc_data executed in 0.24 seconds
+2023-11-28 01:59:31,034 [INFO] - CMC data script execution completed.
+2023-11-28 01:59:31,035 [INFO] - Function fetch_and_process_cmc_data executed in 0.22 seconds
+2023-11-28 01:59:54,297 [INFO] - CMC data script execution completed.
+2023-11-28 01:59:54,298 [INFO] - Function fetch_and_process_cmc_data executed in 0.24 seconds

logs/scrapping/etherscan_scrap.log

@@ -203,3 +203,143 @@
 2023-11-28 00:46:02,648 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
 2023-11-28 00:46:02,648 [INFO] - Etherscan scraping script execution completed.
 2023-11-28 00:46:02,648 [INFO] - Function fetch_and_update_etherscan executed in 21.07 seconds
+2023-11-28 00:59:24,578 [INFO] - Created files: 
+2023-11-28 00:59:24,578 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 00:59:24,578 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 00:59:24,578 [INFO] - Function fetch_and_update_etherscan executed in 22.10 seconds
+2023-11-28 01:04:41,583 [INFO] - Created files: 
+2023-11-28 01:04:41,583 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 01:04:41,584 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 01:04:41,584 [INFO] - Function fetch_and_update_etherscan executed in 54.07 seconds
+2023-11-28 01:05:01,098 [INFO] - Created files: 
+2023-11-28 01:05:01,098 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 01:05:01,098 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 01:05:01,099 [INFO] - Function fetch_and_update_etherscan executed in 29.43 seconds
+2023-11-28 01:05:55,824 [INFO] - Created files: 
+2023-11-28 01:05:55,824 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 01:05:55,824 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 01:05:55,824 [INFO] - Function fetch_and_update_etherscan executed in 21.91 seconds
+2023-11-28 01:06:51,141 [INFO] - Created files: 
+2023-11-28 01:06:51,141 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 01:06:51,141 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 01:06:51,142 [INFO] - Function fetch_and_update_etherscan executed in 22.08 seconds
+2023-11-28 01:07:15,653 [INFO] - Created files: 
+2023-11-28 01:07:15,653 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 01:07:15,653 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 01:07:15,653 [INFO] - Function fetch_and_update_etherscan executed in 20.96 seconds
+2023-11-28 01:08:10,150 [INFO] - Created files: 
+2023-11-28 01:08:10,150 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 01:08:10,150 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 01:08:10,150 [INFO] - Function fetch_and_update_etherscan executed in 23.13 seconds
+2023-11-28 01:10:08,745 [INFO] - Created files: 
+2023-11-28 01:10:08,745 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 01:10:08,745 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 01:10:08,745 [INFO] - Function fetch_and_update_etherscan executed in 21.16 seconds
+2023-11-28 01:10:47,337 [INFO] - Created files: 
+2023-11-28 01:10:47,337 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 01:10:47,337 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 01:10:47,337 [INFO] - Function fetch_and_update_etherscan executed in 23.40 seconds
+2023-11-28 01:11:39,764 [INFO] - Created files: 
+2023-11-28 01:11:39,764 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 01:11:39,764 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 01:11:39,764 [INFO] - Function fetch_and_update_etherscan executed in 21.46 seconds
+2023-11-28 01:13:49,085 [INFO] - Created files: 
+2023-11-28 01:13:49,085 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 01:13:49,085 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 01:13:49,085 [INFO] - Function fetch_and_update_etherscan executed in 20.92 seconds
+2023-11-28 01:16:55,044 [INFO] - Created files: 
+2023-11-28 01:16:55,045 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 01:16:55,045 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 01:16:55,045 [INFO] - Function fetch_and_update_etherscan executed in 22.37 seconds
+2023-11-28 01:17:42,300 [INFO] - Created files: 
+2023-11-28 01:17:42,300 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 01:17:42,300 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 01:17:42,300 [INFO] - Function fetch_and_update_etherscan executed in 21.96 seconds
+2023-11-28 01:18:02,578 [INFO] - Created files: 
+2023-11-28 01:18:02,578 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 01:18:02,578 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 01:18:02,578 [INFO] - Function fetch_and_update_etherscan executed in 21.09 seconds
+2023-11-28 01:25:38,961 [INFO] - Created files: 
+2023-11-28 01:25:38,961 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 01:25:38,961 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 01:25:38,961 [INFO] - Function fetch_and_update_etherscan executed in 22.07 seconds
+2023-11-28 01:27:33,868 [INFO] - Created files: 
+2023-11-28 01:27:33,868 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 01:27:33,868 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 01:27:33,868 [INFO] - Function fetch_and_update_etherscan executed in 22.21 seconds
+2023-11-28 01:31:40,438 [INFO] - Created files: 
+2023-11-28 01:31:40,438 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 01:31:40,438 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 01:31:40,438 [INFO] - Function fetch_and_update_etherscan executed in 21.05 seconds
+2023-11-28 01:33:09,508 [INFO] - Created files: 
+2023-11-28 01:33:09,508 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 01:33:09,508 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 01:33:09,508 [INFO] - Function fetch_and_update_etherscan executed in 21.73 seconds
+2023-11-28 01:36:21,589 [ERROR] - Invalid data format for transactions: Max rate limit reached
+2023-11-28 01:36:21,589 [ERROR] - 'timeStamp' key not found in the response data.
+2023-11-28 01:36:43,768 [INFO] - Created files: 
+2023-11-28 01:36:43,768 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 01:36:43,768 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 01:36:43,768 [INFO] - Function fetch_and_update_etherscan executed in 23.15 seconds
+2023-11-28 01:44:58,251 [INFO] - Created files: 
+2023-11-28 01:44:58,251 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 01:44:58,251 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 01:44:58,251 [INFO] - Function fetch_and_update_etherscan executed in 22.33 seconds
+2023-11-28 01:45:41,270 [INFO] - Created files: 
+2023-11-28 01:45:41,270 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 01:45:41,270 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 01:45:41,270 [INFO] - Function fetch_and_update_etherscan executed in 21.04 seconds
+2023-11-28 01:46:02,127 [INFO] - Created files: 
+2023-11-28 01:46:02,127 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 01:46:02,127 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 01:46:02,127 [INFO] - Function fetch_and_update_etherscan executed in 21.01 seconds
+2023-11-28 01:47:13,494 [INFO] - Created files: 
+2023-11-28 01:47:13,494 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 01:47:13,494 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 01:47:13,494 [INFO] - Function fetch_and_update_etherscan executed in 27.34 seconds
+2023-11-28 01:47:49,201 [INFO] - Created files: 
+2023-11-28 01:47:49,202 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 01:47:49,202 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 01:47:49,202 [INFO] - Function fetch_and_update_etherscan executed in 20.93 seconds
+2023-11-28 01:50:38,362 [INFO] - Created files: 
+2023-11-28 01:50:38,363 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 01:50:38,363 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 01:50:38,363 [INFO] - Function fetch_and_update_etherscan executed in 21.34 seconds
+2023-11-28 01:51:02,371 [INFO] - Created files: 
+2023-11-28 01:51:02,371 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 01:51:02,371 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 01:51:02,371 [INFO] - Function fetch_and_update_etherscan executed in 22.05 seconds
+2023-11-28 01:52:06,254 [INFO] - Created files: 
+2023-11-28 01:52:06,254 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 01:52:06,254 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 01:52:06,254 [INFO] - Function fetch_and_update_etherscan executed in 21.05 seconds
+2023-11-28 01:53:17,208 [ERROR] - Invalid data format for transactions: Max rate limit reached
+2023-11-28 01:53:17,208 [ERROR] - 'timeStamp' key not found in the response data.
+2023-11-28 01:53:23,747 [INFO] - Created files: 
+2023-11-28 01:53:23,747 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 01:53:23,747 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 01:53:23,747 [INFO] - Function fetch_and_update_etherscan executed in 20.98 seconds
+2023-11-28 01:54:24,168 [INFO] - Created files: 
+2023-11-28 01:54:24,168 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 01:54:24,168 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 01:54:24,169 [INFO] - Function fetch_and_update_etherscan executed in 23.05 seconds
+2023-11-28 01:57:14,360 [INFO] - Created files: 
+2023-11-28 01:57:14,360 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 01:57:14,360 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 01:57:14,360 [INFO] - Function fetch_and_update_etherscan executed in 21.15 seconds
+2023-11-28 01:58:17,129 [INFO] - Created files: 
+2023-11-28 01:58:17,129 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 01:58:17,129 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 01:58:17,129 [INFO] - Function fetch_and_update_etherscan executed in 21.19 seconds
+2023-11-28 01:59:06,792 [INFO] - Created files: 
+2023-11-28 01:59:06,792 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 01:59:06,792 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 01:59:06,793 [INFO] - Function fetch_and_update_etherscan executed in 21.07 seconds
+2023-11-28 01:59:32,211 [INFO] - Created files: 
+2023-11-28 01:59:32,211 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 01:59:32,211 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 01:59:32,211 [INFO] - Function fetch_and_update_etherscan executed in 20.96 seconds
+2023-11-28 01:59:52,774 [INFO] - Created files: 
+2023-11-28 01:59:52,774 [INFO] - Updated files: output/transactions_APE.csv, output/transactions_AXIE.csv, output/transactions_GALA.csv, output/transactions_MANA.csv, output/transactions_PET.csv, output/transactions_WEAOPON.csv
+2023-11-28 01:59:52,774 [INFO] - Etherscan scraping script execution completed.
+2023-11-28 01:59:52,774 [INFO] - Function fetch_and_update_etherscan executed in 21.97 seconds

logs/scrapping/greed_fear_index.log

@@ -0,0 +1,23 @@
+2023-11-28 01:25:17,773 [INFO] - Greed and Fear index data fetched successfully.
+2023-11-28 01:27:11,999 [INFO] - Greed and Fear index data fetched successfully.
+2023-11-28 01:31:19,874 [INFO] - Greed and Fear index data fetched successfully.
+2023-11-28 01:32:48,045 [INFO] - Greed and Fear index data fetched successfully.
+2023-11-28 01:35:41,798 [INFO] - Greed and Fear index data fetched successfully.
+2023-11-28 01:35:54,600 [INFO] - Greed and Fear index data fetched successfully.
+2023-11-28 01:36:20,870 [INFO] - Greed and Fear index data fetched successfully.
+2023-11-28 01:44:36,204 [INFO] - Greed and Fear index data fetched successfully.
+2023-11-28 01:45:20,580 [INFO] - Greed and Fear index data fetched successfully.
+2023-11-28 01:45:41,420 [INFO] - Greed and Fear index data fetched successfully.
+2023-11-28 01:46:46,440 [INFO] - Greed and Fear index data fetched successfully.
+2023-11-28 01:47:28,478 [INFO] - Greed and Fear index data fetched successfully.
+2023-11-28 01:50:17,378 [INFO] - Greed and Fear index data fetched successfully.
+2023-11-28 01:50:40,569 [INFO] - Greed and Fear index data fetched successfully.
+2023-11-28 01:51:45,671 [INFO] - Greed and Fear index data fetched successfully.
+2023-11-28 01:53:03,025 [INFO] - Greed and Fear index data fetched successfully.
+2023-11-28 01:53:16,091 [INFO] - Greed and Fear index data fetched successfully.
+2023-11-28 01:54:01,931 [INFO] - Greed and Fear index data fetched successfully.
+2023-11-28 01:56:53,527 [INFO] - Greed and Fear index data fetched successfully.
+2023-11-28 01:57:56,221 [INFO] - Greed and Fear index data fetched successfully.
+2023-11-28 01:58:46,004 [INFO] - Greed and Fear index data fetched successfully.
+2023-11-28 01:59:11,591 [INFO] - Greed and Fear index data fetched successfully.
+2023-11-28 01:59:31,101 [INFO] - Greed and Fear index data fetched successfully.

logs/streamlit/front.log

@@ -369,3 +369,307 @@
 2023-11-28 00:47:28,807 [INFO] - Balances uploaded, shape of dataframe is (4, 2)
 2023-11-28 00:48:02,170 [INFO] - CMC scraping completed
 2023-11-28 00:48:07,741 [INFO] - CMC scraping completed
+2023-11-28 00:59:02,040 [INFO] - Streamlit app has started
+2023-11-28 00:59:02,461 [INFO] - Balances uploaded, shape of dataframe is (4, 2)
+2023-11-28 00:59:02,724 [INFO] - Global metrics scraping completed
+2023-11-28 00:59:02,759 [INFO] - CMC scraping completed
+2023-11-28 00:59:03,013 [INFO] - Influencers balance scraping completed
+2023-11-28 00:59:24,613 [INFO] - Etherscan scraping completed
+2023-11-28 01:03:47,077 [INFO] - Streamlit app has started
+2023-11-28 01:03:47,268 [INFO] - Balances uploaded, shape of dataframe is (4, 2)
+2023-11-28 01:03:47,766 [INFO] - Global metrics scraping completed
+2023-11-28 01:03:47,994 [INFO] - CMC scraping completed
+2023-11-28 01:03:48,030 [INFO] - Influencers balance scraping completed
+2023-11-28 01:03:51,304 [INFO] - CMC scraping completed
+2023-11-28 01:03:51,462 [INFO] - Influencers balance scraping completed
+2023-11-28 01:04:31,222 [INFO] - Streamlit app has started
+2023-11-28 01:04:31,385 [INFO] - Balances uploaded, shape of dataframe is (4, 2)
+2023-11-28 01:04:31,890 [INFO] - Global metrics scraping completed
+2023-11-28 01:04:31,972 [INFO] - CMC scraping completed
+2023-11-28 01:04:39,211 [INFO] - Influencers balance scraping completed
+2023-11-28 01:04:41,624 [INFO] - Etherscan scraping completed
+2023-11-28 01:05:01,141 [INFO] - Etherscan scraping completed
+2023-11-28 01:05:33,429 [INFO] - Streamlit app has started
+2023-11-28 01:05:33,618 [INFO] - Balances uploaded, shape of dataframe is (4, 2)
+2023-11-28 01:05:34,166 [INFO] - Global metrics scraping completed
+2023-11-28 01:05:34,208 [INFO] - CMC scraping completed
+2023-11-28 01:05:34,454 [INFO] - Influencers balance scraping completed
+2023-11-28 01:05:55,862 [INFO] - Etherscan scraping completed
+2023-11-28 01:06:28,647 [INFO] - Streamlit app has started
+2023-11-28 01:06:28,837 [INFO] - Balances uploaded, shape of dataframe is (4, 2)
+2023-11-28 01:06:29,366 [INFO] - Global metrics scraping completed
+2023-11-28 01:06:29,391 [INFO] - CMC scraping completed
+2023-11-28 01:06:29,600 [INFO] - Influencers balance scraping completed
+2023-11-28 01:06:51,183 [INFO] - Etherscan scraping completed
+2023-11-28 01:06:54,258 [INFO] - Streamlit app has started
+2023-11-28 01:06:54,426 [INFO] - Balances uploaded, shape of dataframe is (4, 2)
+2023-11-28 01:06:54,933 [INFO] - Global metrics scraping completed
+2023-11-28 01:06:54,966 [INFO] - CMC scraping completed
+2023-11-28 01:06:55,216 [INFO] - Influencers balance scraping completed
+2023-11-28 01:07:15,694 [INFO] - Etherscan scraping completed
+2023-11-28 01:07:46,534 [INFO] - Streamlit app has started
+2023-11-28 01:07:46,743 [INFO] - Balances uploaded, shape of dataframe is (4, 2)
+2023-11-28 01:07:47,261 [INFO] - Global metrics scraping completed
+2023-11-28 01:07:47,369 [INFO] - CMC scraping completed
+2023-11-28 01:07:47,536 [INFO] - Influencers balance scraping completed
+2023-11-28 01:08:10,193 [INFO] - Etherscan scraping completed
+2023-11-28 01:08:37,541 [INFO] - CMC scraping completed
+2023-11-28 01:08:37,590 [INFO] - Influencers balance scraping completed
+2023-11-28 01:08:39,744 [INFO] - CMC scraping completed
+2023-11-28 01:09:20,480 [INFO] - CMC scraping completed
+2023-11-28 01:09:20,703 [INFO] - Influencers balance scraping completed
+2023-11-28 01:09:47,119 [INFO] - Streamlit app has started
+2023-11-28 01:09:47,308 [INFO] - Balances uploaded, shape of dataframe is (4, 2)
+2023-11-28 01:09:47,881 [INFO] - CMC scraping completed
+2023-11-28 01:09:48,058 [INFO] - Global metrics scraping completed
+2023-11-28 01:09:48,116 [INFO] - Influencers balance scraping completed
+2023-11-28 01:10:08,784 [INFO] - Etherscan scraping completed
+2023-11-28 01:10:22,735 [INFO] - CMC scraping completed
+2023-11-28 01:10:22,957 [INFO] - Influencers balance scraping completed
+2023-11-28 01:10:23,478 [INFO] - Streamlit app has started
+2023-11-28 01:10:23,652 [INFO] - Balances uploaded, shape of dataframe is (4, 2)
+2023-11-28 01:10:24,143 [INFO] - Global metrics scraping completed
+2023-11-28 01:10:24,232 [INFO] - CMC scraping completed
+2023-11-28 01:10:27,463 [INFO] - Influencers balance scraping completed
+2023-11-28 01:10:47,374 [INFO] - Etherscan scraping completed
+2023-11-28 01:11:17,841 [INFO] - Streamlit app has started
+2023-11-28 01:11:18,004 [INFO] - Balances uploaded, shape of dataframe is (4, 2)
+2023-11-28 01:11:18,194 [INFO] - Influencers balance scraping completed
+2023-11-28 01:11:18,492 [INFO] - Global metrics scraping completed
+2023-11-28 01:11:18,779 [INFO] - Influencers balance scraping completed
+2023-11-28 01:11:18,881 [INFO] - CMC scraping completed
+2023-11-28 01:11:19,248 [INFO] - CMC scraping completed
+2023-11-28 01:11:39,806 [INFO] - Etherscan scraping completed
+2023-11-28 01:11:43,490 [INFO] - CMC scraping completed
+2023-11-28 01:11:43,702 [INFO] - Influencers balance scraping completed
+2023-11-28 01:12:35,836 [INFO] - CMC scraping completed
+2023-11-28 01:12:35,992 [INFO] - Influencers balance scraping completed
+2023-11-28 01:13:26,151 [INFO] - CMC scraping completed
+2023-11-28 01:13:27,674 [INFO] - Streamlit app has started
+2023-11-28 01:13:27,877 [INFO] - Balances uploaded, shape of dataframe is (4, 2)
+2023-11-28 01:13:28,405 [INFO] - Global metrics scraping completed
+2023-11-28 01:13:28,518 [INFO] - CMC scraping completed
+2023-11-28 01:13:28,563 [INFO] - CMC scraping completed
+2023-11-28 01:13:28,715 [INFO] - Influencers balance scraping completed
+2023-11-28 01:13:49,131 [INFO] - Etherscan scraping completed
+2023-11-28 01:14:09,193 [INFO] - CMC scraping completed
+2023-11-28 01:14:36,376 [INFO] - CMC scraping completed
+2023-11-28 01:14:36,768 [INFO] - Influencers balance scraping completed
+2023-11-28 01:15:11,208 [INFO] - CMC scraping completed
+2023-11-28 01:15:12,683 [INFO] - CMC scraping completed
+2023-11-28 01:15:12,929 [INFO] - Influencers balance scraping completed
+2023-11-28 01:16:07,206 [INFO] - Influencers balance scraping completed
+2023-11-28 01:16:07,375 [INFO] - CMC scraping completed
+2023-11-28 01:16:07,705 [INFO] - CMC scraping completed
+2023-11-28 01:16:32,017 [INFO] - Streamlit app has started
+2023-11-28 01:16:32,725 [INFO] - Balances uploaded, shape of dataframe is (4, 2)
+2023-11-28 01:16:32,895 [INFO] - Global metrics scraping completed
+2023-11-28 01:16:32,964 [INFO] - CMC scraping completed
+2023-11-28 01:16:33,170 [INFO] - Influencers balance scraping completed
+2023-11-28 01:16:55,083 [INFO] - Etherscan scraping completed
+2023-11-28 01:17:19,882 [INFO] - Streamlit app has started
+2023-11-28 01:17:20,039 [INFO] - Balances uploaded, shape of dataframe is (4, 2)
+2023-11-28 01:17:20,579 [INFO] - Global metrics scraping completed
+2023-11-28 01:17:20,692 [INFO] - CMC scraping completed
+2023-11-28 01:17:20,857 [INFO] - Influencers balance scraping completed
+2023-11-28 01:17:41,042 [INFO] - Streamlit app has started
+2023-11-28 01:17:41,220 [INFO] - Balances uploaded, shape of dataframe is (4, 2)
+2023-11-28 01:17:41,724 [INFO] - Global metrics scraping completed
+2023-11-28 01:17:41,974 [INFO] - Influencers balance scraping completed
+2023-11-28 01:17:42,234 [INFO] - CMC scraping completed
+2023-11-28 01:17:42,336 [INFO] - Etherscan scraping completed
+2023-11-28 01:18:02,615 [INFO] - Etherscan scraping completed
+2023-11-28 01:21:21,525 [INFO] - CMC scraping completed
+2023-11-28 01:21:21,715 [INFO] - Influencers balance scraping completed
+2023-11-28 01:22:10,446 [INFO] - CMC scraping completed
+2023-11-28 01:22:10,532 [INFO] - Influencers balance scraping completed
+2023-11-28 01:22:30,474 [INFO] - Influencers balance scraping completed
+2023-11-28 01:22:30,742 [INFO] - CMC scraping completed
+2023-11-28 01:25:16,428 [INFO] - Streamlit app has started
+2023-11-28 01:25:16,580 [INFO] - Balances uploaded, shape of dataframe is (4, 2)
+2023-11-28 01:25:17,140 [INFO] - Global metrics scraping completed
+2023-11-28 01:25:17,197 [INFO] - CMC scraping completed
+2023-11-28 01:25:17,425 [INFO] - Influencers balance scraping completed
+2023-11-28 01:25:17,807 [INFO] - Greed and Fear index scraping completed
+2023-11-28 01:25:38,997 [INFO] - Etherscan scraping completed
+2023-11-28 01:26:10,003 [INFO] - CMC scraping completed
+2023-11-28 01:26:58,934 [INFO] - CMC scraping completed
+2023-11-28 01:27:11,201 [INFO] - Streamlit app has started
+2023-11-28 01:27:11,348 [INFO] - Balances uploaded, shape of dataframe is (4, 2)
+2023-11-28 01:27:11,900 [INFO] - Global metrics scraping completed
+2023-11-28 01:27:11,971 [INFO] - CMC scraping completed
+2023-11-28 01:27:12,050 [INFO] - Greed and Fear index scraping completed
+2023-11-28 01:27:12,206 [INFO] - Influencers balance scraping completed
+2023-11-28 01:27:19,260 [INFO] - CMC scraping completed
+2023-11-28 01:27:33,910 [INFO] - Etherscan scraping completed
+2023-11-28 01:30:05,828 [INFO] - CMC scraping completed
+2023-11-28 01:30:05,943 [INFO] - Influencers balance scraping completed
+2023-11-28 01:30:58,512 [INFO] - CMC scraping completed
+2023-11-28 01:31:18,910 [INFO] - Streamlit app has started
+2023-11-28 01:31:19,414 [INFO] - Balances uploaded, shape of dataframe is (4, 2)
+2023-11-28 01:31:19,648 [INFO] - Global metrics scraping completed
+2023-11-28 01:31:19,687 [INFO] - CMC scraping completed
+2023-11-28 01:31:19,941 [INFO] - Greed and Fear index scraping completed
+2023-11-28 01:31:19,960 [INFO] - Influencers balance scraping completed
+2023-11-28 01:31:40,492 [INFO] - Etherscan scraping completed
+2023-11-28 01:32:47,285 [INFO] - Streamlit app has started
+2023-11-28 01:32:47,454 [INFO] - Balances uploaded, shape of dataframe is (4, 2)
+2023-11-28 01:32:48,004 [INFO] - Global metrics scraping completed
+2023-11-28 01:32:48,057 [INFO] - CMC scraping completed
+2023-11-28 01:32:48,103 [INFO] - Greed and Fear index scraping completed
+2023-11-28 01:32:48,272 [INFO] - Influencers balance scraping completed
+2023-11-28 01:33:09,545 [INFO] - Etherscan scraping completed
+2023-11-28 01:35:40,742 [INFO] - Streamlit app has started
+2023-11-28 01:35:40,940 [INFO] - Balances uploaded, shape of dataframe is (4, 2)
+2023-11-28 01:35:41,431 [INFO] - Global metrics scraping completed
+2023-11-28 01:35:41,489 [INFO] - CMC scraping completed
+2023-11-28 01:35:41,741 [INFO] - Influencers balance scraping completed
+2023-11-28 01:35:41,851 [INFO] - Greed and Fear index scraping completed
+2023-11-28 01:35:50,588 [INFO] - Etherscan scraping completed
+2023-11-28 01:35:53,853 [INFO] - Streamlit app has started
+2023-11-28 01:35:54,354 [INFO] - Balances uploaded, shape of dataframe is (4, 2)
+2023-11-28 01:35:54,569 [INFO] - Global metrics scraping completed
+2023-11-28 01:35:54,654 [INFO] - Greed and Fear index scraping completed
+2023-11-28 01:35:54,859 [INFO] - CMC scraping completed
+2023-11-28 01:35:55,895 [INFO] - Influencers balance scraping completed
+2023-11-28 01:36:20,153 [INFO] - Streamlit app has started
+2023-11-28 01:36:20,359 [INFO] - Balances uploaded, shape of dataframe is (4, 2)
+2023-11-28 01:36:20,840 [INFO] - Global metrics scraping completed
+2023-11-28 01:36:20,932 [INFO] - Greed and Fear index scraping completed
+2023-11-28 01:36:21,000 [INFO] - CMC scraping completed
+2023-11-28 01:36:21,129 [INFO] - Influencers balance scraping completed
+2023-11-28 01:36:21,642 [INFO] - Etherscan scraping completed
+2023-11-28 01:36:43,808 [INFO] - Etherscan scraping completed
+2023-11-28 01:40:43,308 [INFO] - Influencers balance scraping completed
+2023-11-28 01:40:43,328 [INFO] - CMC scraping completed
+2023-11-28 01:41:09,539 [INFO] - CMC scraping completed
+2023-11-28 01:41:09,589 [INFO] - Influencers balance scraping completed
+2023-11-28 01:44:35,447 [INFO] - Streamlit app has started
+2023-11-28 01:44:35,614 [INFO] - Balances uploaded, shape of dataframe is (4, 2)
+2023-11-28 01:44:36,126 [INFO] - Global metrics scraping completed
+2023-11-28 01:44:36,184 [INFO] - CMC scraping completed
+2023-11-28 01:44:36,254 [INFO] - Greed and Fear index scraping completed
+2023-11-28 01:44:36,427 [INFO] - Influencers balance scraping completed
+2023-11-28 01:44:58,286 [INFO] - Etherscan scraping completed
+2023-11-28 01:45:19,733 [INFO] - Streamlit app has started
+2023-11-28 01:45:19,931 [INFO] - Balances uploaded, shape of dataframe is (4, 2)
+2023-11-28 01:45:20,441 [INFO] - Global metrics scraping completed
+2023-11-28 01:45:20,525 [INFO] - CMC scraping completed
+2023-11-28 01:45:20,631 [INFO] - Greed and Fear index scraping completed
+2023-11-28 01:45:20,723 [INFO] - Influencers balance scraping completed
+2023-11-28 01:45:31,803 [INFO] - CMC scraping completed
+2023-11-28 01:45:40,640 [INFO] - Streamlit app has started
+2023-11-28 01:45:40,840 [INFO] - Balances uploaded, shape of dataframe is (4, 2)
+2023-11-28 01:45:41,336 [INFO] - Global metrics scraping completed
+2023-11-28 01:45:41,359 [INFO] - Etherscan scraping completed
+2023-11-28 01:45:41,476 [INFO] - CMC scraping completed
+2023-11-28 01:45:41,499 [INFO] - Greed and Fear index scraping completed
+2023-11-28 01:45:41,628 [INFO] - Influencers balance scraping completed
+2023-11-28 01:45:58,273 [INFO] - CMC scraping completed
+2023-11-28 01:46:02,165 [INFO] - Etherscan scraping completed
+2023-11-28 01:46:45,670 [INFO] - Streamlit app has started
+2023-11-28 01:46:45,842 [INFO] - Balances uploaded, shape of dataframe is (4, 2)
+2023-11-28 01:46:46,334 [INFO] - Global metrics scraping completed
+2023-11-28 01:46:46,394 [INFO] - CMC scraping completed
+2023-11-28 01:46:46,497 [INFO] - Greed and Fear index scraping completed
+2023-11-28 01:46:52,732 [INFO] - Influencers balance scraping completed
+2023-11-28 01:47:13,543 [INFO] - Etherscan scraping completed
+2023-11-28 01:47:27,785 [INFO] - Streamlit app has started
+2023-11-28 01:47:28,287 [INFO] - Balances uploaded, shape of dataframe is (4, 2)
+2023-11-28 01:47:28,549 [INFO] - Global metrics scraping completed
+2023-11-28 01:47:28,580 [INFO] - Greed and Fear index scraping completed
+2023-11-28 01:47:28,584 [INFO] - CMC scraping completed
+2023-11-28 01:47:28,796 [INFO] - Influencers balance scraping completed
+2023-11-28 01:47:49,239 [INFO] - Etherscan scraping completed
+2023-11-28 01:50:16,530 [INFO] - Streamlit app has started
+2023-11-28 01:50:16,683 [INFO] - Balances uploaded, shape of dataframe is (4, 2)
+2023-11-28 01:50:17,245 [INFO] - Global metrics scraping completed
+2023-11-28 01:50:17,315 [INFO] - CMC scraping completed
+2023-11-28 01:50:17,432 [INFO] - Greed and Fear index scraping completed
+2023-11-28 01:50:17,575 [INFO] - Influencers balance scraping completed
+2023-11-28 01:50:38,407 [INFO] - Etherscan scraping completed
+2023-11-28 01:50:39,844 [INFO] - Streamlit app has started
+2023-11-28 01:50:40,008 [INFO] - Balances uploaded, shape of dataframe is (4, 2)
+2023-11-28 01:50:40,517 [INFO] - Global metrics scraping completed
+2023-11-28 01:50:40,607 [INFO] - CMC scraping completed
+2023-11-28 01:50:40,641 [INFO] - Greed and Fear index scraping completed
+2023-11-28 01:50:40,829 [INFO] - Influencers balance scraping completed
+2023-11-28 01:51:02,408 [INFO] - Etherscan scraping completed
+2023-11-28 01:51:44,739 [INFO] - Streamlit app has started
+2023-11-28 01:51:44,933 [INFO] - Balances uploaded, shape of dataframe is (4, 2)
+2023-11-28 01:51:45,453 [INFO] - Global metrics scraping completed
+2023-11-28 01:51:45,540 [INFO] - CMC scraping completed
+2023-11-28 01:51:45,718 [INFO] - Influencers balance scraping completed
+2023-11-28 01:51:45,721 [INFO] - Greed and Fear index scraping completed
+2023-11-28 01:52:06,295 [INFO] - Etherscan scraping completed
+2023-11-28 01:52:17,088 [INFO] - CMC scraping completed
+2023-11-28 01:52:17,297 [INFO] - Influencers balance scraping completed
+2023-11-28 01:53:02,317 [INFO] - Streamlit app has started
+2023-11-28 01:53:02,465 [INFO] - Balances uploaded, shape of dataframe is (4, 2)
+2023-11-28 01:53:03,026 [INFO] - Global metrics scraping completed
+2023-11-28 01:53:03,094 [INFO] - Greed and Fear index scraping completed
+2023-11-28 01:53:03,094 [INFO] - CMC scraping completed
+2023-11-28 01:53:03,292 [INFO] - Influencers balance scraping completed
+2023-11-28 01:53:15,335 [INFO] - Streamlit app has started
+2023-11-28 01:53:15,540 [INFO] - Balances uploaded, shape of dataframe is (4, 2)
+2023-11-28 01:53:16,041 [INFO] - Global metrics scraping completed
+2023-11-28 01:53:16,119 [INFO] - CMC scraping completed
+2023-11-28 01:53:16,156 [INFO] - Greed and Fear index scraping completed
+2023-11-28 01:53:16,332 [INFO] - Influencers balance scraping completed
+2023-11-28 01:53:17,254 [INFO] - Etherscan scraping completed
+2023-11-28 01:53:23,795 [INFO] - Etherscan scraping completed
+2023-11-28 01:54:00,654 [INFO] - Streamlit app has started
+2023-11-28 01:54:00,795 [INFO] - Balances uploaded, shape of dataframe is (4, 2)
+2023-11-28 01:54:01,376 [INFO] - CMC scraping completed
+2023-11-28 01:54:01,486 [INFO] - Global metrics scraping completed
+2023-11-28 01:54:01,627 [INFO] - Influencers balance scraping completed
+2023-11-28 01:54:01,982 [INFO] - Greed and Fear index scraping completed
+2023-11-28 01:54:24,210 [INFO] - Etherscan scraping completed
+2023-11-28 01:55:05,841 [INFO] - CMC scraping completed
+2023-11-28 01:55:06,006 [INFO] - Influencers balance scraping completed
+2023-11-28 01:55:29,184 [INFO] - CMC scraping completed
+2023-11-28 01:55:29,271 [INFO] - Influencers balance scraping completed
+2023-11-28 01:56:34,119 [INFO] - CMC scraping completed
+2023-11-28 01:56:34,248 [INFO] - Influencers balance scraping completed
+2023-11-28 01:56:52,756 [INFO] - Streamlit app has started
+2023-11-28 01:56:52,917 [INFO] - Balances uploaded, shape of dataframe is (4, 2)
+2023-11-28 01:56:53,440 [INFO] - Global metrics scraping completed
+2023-11-28 01:56:53,524 [INFO] - CMC scraping completed
+2023-11-28 01:56:53,581 [INFO] - Greed and Fear index scraping completed
+2023-11-28 01:56:53,734 [INFO] - Influencers balance scraping completed
+2023-11-28 01:57:05,559 [INFO] - CMC scraping completed
+2023-11-28 01:57:14,399 [INFO] - Etherscan scraping completed
+2023-11-28 01:57:51,598 [INFO] - CMC scraping completed
+2023-11-28 01:57:51,745 [INFO] - Influencers balance scraping completed
+2023-11-28 01:57:55,477 [INFO] - Streamlit app has started
+2023-11-28 01:57:55,651 [INFO] - Balances uploaded, shape of dataframe is (4, 2)
+2023-11-28 01:57:56,180 [INFO] - Global metrics scraping completed
+2023-11-28 01:57:56,263 [INFO] - CMC scraping completed
+2023-11-28 01:57:56,279 [INFO] - Greed and Fear index scraping completed
+2023-11-28 01:57:56,493 [INFO] - Influencers balance scraping completed
+2023-11-28 01:58:04,644 [INFO] - CMC scraping completed
+2023-11-28 01:58:04,776 [INFO] - Influencers balance scraping completed
+2023-11-28 01:58:17,167 [INFO] - Etherscan scraping completed
+2023-11-28 01:58:45,266 [INFO] - Streamlit app has started
+2023-11-28 01:58:45,467 [INFO] - Balances uploaded, shape of dataframe is (4, 2)
+2023-11-28 01:58:45,968 [INFO] - Global metrics scraping completed
+2023-11-28 01:58:46,060 [INFO] - Greed and Fear index scraping completed
+2023-11-28 01:58:46,250 [INFO] - Influencers balance scraping completed
+2023-11-28 01:58:46,292 [INFO] - CMC scraping completed
+2023-11-28 01:58:49,844 [INFO] - CMC scraping completed
+2023-11-28 01:58:50,183 [INFO] - Influencers balance scraping completed
+2023-11-28 01:59:06,843 [INFO] - Etherscan scraping completed
+2023-11-28 01:59:10,767 [INFO] - Streamlit app has started
+2023-11-28 01:59:11,536 [INFO] - CMC scraping completed
+2023-11-28 01:59:11,641 [INFO] - Greed and Fear index scraping completed
+2023-11-28 01:59:11,769 [INFO] - Influencers balance scraping completed
+2023-11-28 01:59:14,144 [INFO] - Global metrics scraping completed
+2023-11-28 01:59:30,345 [INFO] - Streamlit app has started
+2023-11-28 01:59:30,495 [INFO] - Balances uploaded, shape of dataframe is (4, 2)
+2023-11-28 01:59:31,106 [INFO] - Global metrics scraping completed
+2023-11-28 01:59:31,109 [INFO] - CMC scraping completed
+2023-11-28 01:59:31,158 [INFO] - Greed and Fear index scraping completed
+2023-11-28 01:59:31,325 [INFO] - Influencers balance scraping completed
+2023-11-28 01:59:32,265 [INFO] - Etherscan scraping completed
+2023-11-28 01:59:52,816 [INFO] - Etherscan scraping completed
+2023-11-28 01:59:54,340 [INFO] - CMC scraping completed

output/cmc_data_v1_cryptocurrency_listings_latest_100.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5be145c802161fd6ec2f4932bd5c782d41e2c1e166a966bdaf080955e45397eb
-size 119005
+oid sha256:49f394b714b63bcb568bc1f85ea5a8fe308067977994fc77a11d7401f613b2ea
+size 119061

output/global_metrics.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56d354bb20eac786e584d13e27b0648dbfff97321da9cfea34ab2b5a937e956a
+size 2058

output/greed_fear_index.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4eee12b593c936615d9b103bf596fbcb999542a4a68987aff4100b9d5396cf15
+size 363

output/top_100_update.csv

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7e6bafeb039ce6f24f145e6ebbec3eacbec576d432111ce9767686e3a85ccdf3
-size 748449
+oid sha256:c00ca112a17aa4596689a06be0226051c6ef155402a24d5f61d8b4562b3a1d8c
+size 1325534

output/transactions_APE.csv

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fd20aeb6fc41c40b6d62bf00c3921bfa5cc0c803aea1a2817656db5efdebf81c
-size 3457395
+oid sha256:047ff00b9a1610e53ccf954fddcb4dff40cf5805f98090465f82072150590d1c
+size 3491358

output/transactions_AXIE.csv

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e1bcc9db49c214ca61d447a783162785e1e3a2e11cc5b037e8430addfbfd42d9
-size 1610374
+oid sha256:c973cac0473cbbe02c7bdba3704a97a5819fb130118d9cd99b8b1667552b8a60
+size 1646172

output/transactions_GALA.csv

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3c382d5715b1551fe22176130dd9b139d446339819cfe22eb8e55cbdaf391858
-size 4164007
+oid sha256:b52b4143f26394c83aafdce92babe5a91d78e5f47ab824dfbe1e5428caa09be2
+size 4222917

output/transactions_MANA.csv

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:954bb98d20c48c0e62cb3349dbaeb4af1499c5738f664f4434cd170d9351b787
-size 2237025
+oid sha256:da79712ca9bbe25335e2b9c4ce28b9aa551a0a84ff013aa95b8dd22ec3f0bc12
+size 2254698

utils/scrap_cmc_global_metrics.py

@@ -0,0 +1,35 @@
+# Import necessary libraries
+import requests
+import pandas as pd
+import logging
+import os
+from dotenv import load_dotenv
+
+# Load environment variables
+load_dotenv()
+url_cmc = os.getenv("URL_CMC")
+api_key_cmc = os.getenv("API_KEY_CMC")
+log_folder = os.getenv("LOG_FOLDER")
+
+# Ensure the log folder exists
+os.makedirs(log_folder, exist_ok=True)
+log_file = os.path.join(log_folder, "cmc_global_metrics.log")
+logging.basicConfig(filename=log_file, level=logging.INFO, format="%(asctime)s [%(levelname)s] - %(message)s")
+
+# Fetch global metrics
+def fetch_global_metrics():
+    endpoint = "v1/global-metrics/quotes/latest"
+    headers = {'Accepts': 'application/json', 'X-CMC_PRO_API_KEY': api_key_cmc}
+    try:
+        response = requests.get(f"{url_cmc}/{endpoint}", headers=headers)
+        response.raise_for_status()  # Raise an exception for HTTP errors
+        data = response.json()
+        df = pd.json_normalize(data['data'])
+        df.to_csv('output/global_metrics.csv', index=False)
+        logging.info("Global metrics data fetched successfully.")
+    except requests.exceptions.RequestException as e:
+        logging.error(f"Error fetching global metrics data: {e}")
+
+# Run the function
+if __name__ == "__main__":
+    fetch_global_metrics()

utils/scrap_greed_fear_index.py

@@ -0,0 +1,34 @@
+import requests
+import pandas as pd
+import os
+from dotenv import load_dotenv
+import logging
+
+# Load environment variables
+load_dotenv()
+api_key = os.getenv("RAPIDAPI_KEY")
+
+# Setup logging
+log_folder = os.getenv("LOG_FOLDER")
+os.makedirs(log_folder, exist_ok=True)
+log_file = os.path.join(log_folder, "greed_fear_index.log")
+logging.basicConfig(filename=log_file, level=logging.INFO, format="%(asctime)s [%(levelname)s] - %(message)s")
+
+def fetch_greed_fear_index():
+    url = "https://fear-and-greed-index.p.rapidapi.com/v1/fgi"
+    headers = {
+        "X-RapidAPI-Key": api_key,
+        "X-RapidAPI-Host": "fear-and-greed-index.p.rapidapi.com"
+    }
+
+    response = requests.get(url, headers=headers)
+    if response.status_code == 200:
+        data = response.json()
+        df = pd.json_normalize(data, sep='_')
+        df.to_csv('output/greed_fear_index.csv', index=False)
+        logging.info("Greed and Fear index data fetched successfully.")
+    else:
+        logging.error(f"Failed to fetch data: {response.text}")
+
+if __name__ == "__main__":
+    fetch_greed_fear_index()