Spaces:
Sleeping
Sleeping
Commit
•
1a57d8f
1
Parent(s):
8f00147
init
Browse files- .env +3 -0
- .gitattributes +1 -0
- app.py +136 -0
- app_dash.py +73 -0
- output/top_100_update.csv +3 -0
- output/transactions_APE.csv +3 -0
- output/transactions_AXIE.csv +3 -0
- output/transactions_GALA.csv +3 -0
- output/transactions_ILV.csv +3 -0
- output/transactions_MANA.csv +3 -0
- output/transactions_PET.csv +3 -0
- output/transactions_WEAOPON.csv +3 -0
- requirements.txt +12 -0
- scrap_data_CMC.py +88 -0
- scrap_data_etherscan.py +17 -0
.env
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
AIRFLOW_UID=1000
|
2 |
+
URL_CMC=https://pro-api.coinmarketcap.com
|
3 |
+
API_KEY_CMC=8057498e-ad35-465c-8359-8f6cc9d1ae1b
|
.gitattributes
CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
36 |
+
output/* filter=lfs diff=lfs merge=lfs -text
|
app.py
ADDED
@@ -0,0 +1,136 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
#-------------------------------------libraries ----------------------------------
|
2 |
+
|
3 |
+
import os
|
4 |
+
import pandas as pd
|
5 |
+
import streamlit as st
|
6 |
+
import plotly.graph_objs as go
|
7 |
+
import numpy as np
|
8 |
+
import plotly.express as px
|
9 |
+
import logging
|
10 |
+
# Set up logging basic configuration
|
11 |
+
logging.basicConfig(level=logging.INFO)
|
12 |
+
# Example of logging
|
13 |
+
logging.info("Streamlit app has started")
|
14 |
+
|
15 |
+
|
16 |
+
#-------------------------------------back ----------------------------------
|
17 |
+
|
18 |
+
# etherscan
|
19 |
+
## Load the data from the CSV files
|
20 |
+
dataframes = []
|
21 |
+
for filename in os.listdir('output'):
|
22 |
+
if filename.endswith('.csv'):
|
23 |
+
df_temp = pd.read_csv(os.path.join('output', filename), sep=';')
|
24 |
+
dataframes.append(df_temp)
|
25 |
+
df_etherscan = pd.concat(dataframes)
|
26 |
+
del df_temp
|
27 |
+
|
28 |
+
# CMC
|
29 |
+
## Load cmc data
|
30 |
+
df_temp = pd.read_csv("output/top_100_update.csv", sep=',')
|
31 |
+
df_cmc = df_temp[df_temp["last_updated"] == df_temp["last_updated"].max()]
|
32 |
+
del df_temp
|
33 |
+
#-------------------------------------streamlit ----------------------------------
|
34 |
+
|
35 |
+
# Set the title and other page configurations
|
36 |
+
st.title('Crypto Analysis')
|
37 |
+
# Create two columns for the two plots
|
38 |
+
col1, col2 = st.columns(2)
|
39 |
+
|
40 |
+
with st.container():
|
41 |
+
|
42 |
+
with col1:
|
43 |
+
# etherscan
|
44 |
+
selected_token = st.selectbox('Select Token', df_etherscan['tokenSymbol'].unique(), index=0)
|
45 |
+
# Filter the data based on the selected token
|
46 |
+
filtered_df = df_etherscan[df_etherscan['tokenSymbol'] == selected_token]
|
47 |
+
# Plot the token value over time
|
48 |
+
st.plotly_chart(
|
49 |
+
go.Figure(
|
50 |
+
data=[
|
51 |
+
go.Scatter(
|
52 |
+
x=filtered_df['timeStamp'],
|
53 |
+
y=filtered_df['value'],
|
54 |
+
mode='lines',
|
55 |
+
name='Value over time'
|
56 |
+
)
|
57 |
+
],
|
58 |
+
layout=go.Layout(
|
59 |
+
title='Token Value Over Time',
|
60 |
+
yaxis=dict(
|
61 |
+
title=f'Value ({selected_token})',
|
62 |
+
),
|
63 |
+
showlegend=True,
|
64 |
+
legend=go.layout.Legend(x=0, y=1.0),
|
65 |
+
margin=go.layout.Margin(l=40, r=0, t=40, b=30),
|
66 |
+
width=500,
|
67 |
+
height=500
|
68 |
+
|
69 |
+
)
|
70 |
+
)
|
71 |
+
)
|
72 |
+
|
73 |
+
with col2:
|
74 |
+
# cmc
|
75 |
+
selected_var = st.selectbox('Select Token', ["percent_change_24h","percent_change_7d","percent_change_90d"], index=0)
|
76 |
+
# Sort the DataFrame by the 'percent_change_24h' column in ascending order
|
77 |
+
df_sorted = df_cmc.sort_values(by=selected_var, ascending=False)
|
78 |
+
# Select the top 10 and worst 10 rows
|
79 |
+
top_10 = df_sorted.head(10)
|
80 |
+
worst_10 = df_sorted.tail(10)
|
81 |
+
# Combine the top and worst dataframes for plotting
|
82 |
+
combined_df = pd.concat([top_10, worst_10], axis=0)
|
83 |
+
max_abs_val = max(abs(combined_df[selected_var].min()), abs(combined_df[selected_var].max()))
|
84 |
+
|
85 |
+
# Create a bar plot for the top 10 with a green color scale
|
86 |
+
fig = go.Figure(data=[
|
87 |
+
go.Bar(
|
88 |
+
x=top_10["symbol"],
|
89 |
+
y=top_10[selected_var],
|
90 |
+
marker_color='rgb(0,100,0)', # Green color for top 10
|
91 |
+
hovertext= "Name : "+top_10["name"].astype(str)+ '<br>' +
|
92 |
+
selected_var + " : " + top_10["percent_tokens_circulation"].astype(str) + '<br>' +
|
93 |
+
'Market Cap: ' + top_10["market_cap"].astype(str) + '<br>' +
|
94 |
+
'Fully Diluted Market Cap: ' + top_10["fully_diluted_market_cap"].astype(str) + '<br>' +
|
95 |
+
'Last Updated: ' + top_10["last_updated"].astype(str),
|
96 |
+
name="top_10"
|
97 |
+
)
|
98 |
+
])
|
99 |
+
|
100 |
+
# Add the worst 10 to the same plot with a red color scale
|
101 |
+
fig.add_traces(go.Bar(
|
102 |
+
x=worst_10["symbol"],
|
103 |
+
y=worst_10[selected_var],
|
104 |
+
marker_color='rgb(255,0,0)', # Red color for worst 10
|
105 |
+
hovertext="Name:"+worst_10["name"].astype(str)+ '<br>' +
|
106 |
+
selected_var + " : " + worst_10["percent_tokens_circulation"].astype(str) + '<br>' +
|
107 |
+
'Market Cap: ' + worst_10["market_cap"].astype(str) + '<br>' +
|
108 |
+
'Fully Diluted Market Cap: ' + worst_10["fully_diluted_market_cap"].astype(str) + '<br>' +
|
109 |
+
'Last Updated: ' + worst_10["last_updated"].astype(str),
|
110 |
+
name="worst_10"
|
111 |
+
)
|
112 |
+
)
|
113 |
+
|
114 |
+
# Customize aspect
|
115 |
+
fig.update_traces(marker_line_color='rgb(8,48,107)', marker_line_width=1.5, opacity=0.8)
|
116 |
+
fig.update_layout(title_text=f'Top 10 and Worst 10 by {selected_var.split("_")[-1]} Percentage Change')
|
117 |
+
fig.update_xaxes(categoryorder='total ascending')
|
118 |
+
fig.update_layout(
|
119 |
+
autosize=False,
|
120 |
+
width=500,
|
121 |
+
height=500,
|
122 |
+
margin=dict(
|
123 |
+
l=50,
|
124 |
+
r=50,
|
125 |
+
b=100,
|
126 |
+
t=100,
|
127 |
+
pad=4
|
128 |
+
),
|
129 |
+
#paper_bgcolor="LightSteelBlue",
|
130 |
+
)
|
131 |
+
st.plotly_chart(fig)
|
132 |
+
|
133 |
+
|
134 |
+
|
135 |
+
|
136 |
+
#-------------------------------------end ----------------------------------
|
app_dash.py
ADDED
@@ -0,0 +1,73 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import os
|
2 |
+
import pandas as pd
|
3 |
+
import dash
|
4 |
+
from dash import dcc,html
|
5 |
+
import dash_bootstrap_components as dbc
|
6 |
+
from dash.dependencies import Input, Output
|
7 |
+
import plotly.graph_objs as go
|
8 |
+
|
9 |
+
# Load the data from the CSV files
|
10 |
+
dataframes = []
|
11 |
+
for filename in os.listdir('output'):
|
12 |
+
if filename.endswith('.csv'):
|
13 |
+
df = pd.read_csv(os.path.join('output', filename), sep=';')
|
14 |
+
dataframes.append(df)
|
15 |
+
df = pd.concat(dataframes)
|
16 |
+
|
17 |
+
|
18 |
+
# Create the Dash app
|
19 |
+
app = dash.Dash(__name__, external_stylesheets=[dbc.themes.BOOTSTRAP])
|
20 |
+
|
21 |
+
# Define the app layout
|
22 |
+
app.layout = dbc.Container([
|
23 |
+
dbc.Row([
|
24 |
+
dbc.Col([
|
25 |
+
html.H1('Token Analysis'),
|
26 |
+
dcc.Dropdown(
|
27 |
+
id='token-dropdown',
|
28 |
+
options=[{'label': i, 'value': i} for i in df['tokenSymbol'].unique()],
|
29 |
+
value='MANA'
|
30 |
+
),
|
31 |
+
# Add more filters here
|
32 |
+
], width=5),
|
33 |
+
dbc.Col([
|
34 |
+
dcc.Graph(id='token-graph')
|
35 |
+
], width=7)
|
36 |
+
])
|
37 |
+
])
|
38 |
+
|
39 |
+
# Define the callback to update the graph
|
40 |
+
@app.callback(
|
41 |
+
Output('token-graph', 'figure'),
|
42 |
+
[Input('token-dropdown', 'value')]
|
43 |
+
)
|
44 |
+
def update_graph(selected_token):
|
45 |
+
filtered_df = df[df['tokenSymbol'] == selected_token]
|
46 |
+
# filtered_df['timeStamp'] = pd.to_datetime(filtered_df['timeStamp'], unit='s')
|
47 |
+
# filtered_df['value'] = filtered_df['value'].astype(float) / 1e18
|
48 |
+
figure = go.Figure(
|
49 |
+
data=[
|
50 |
+
go.Scatter(
|
51 |
+
x=filtered_df['timeStamp'],
|
52 |
+
y=filtered_df['value'],
|
53 |
+
mode='lines',
|
54 |
+
name='Value over time'
|
55 |
+
)
|
56 |
+
],
|
57 |
+
layout=go.Layout(
|
58 |
+
title='Token Value Over Time',
|
59 |
+
yaxis=dict(
|
60 |
+
title='Value ('+selected_token+')', # Change this to 'Value (USD)' if the values are in USD
|
61 |
+
),
|
62 |
+
showlegend=True,
|
63 |
+
legend=go.layout.Legend(
|
64 |
+
x=0,
|
65 |
+
y=1.0
|
66 |
+
),
|
67 |
+
margin=go.layout.Margin(l=40, r=0, t=40, b=30)
|
68 |
+
)
|
69 |
+
)
|
70 |
+
return figure
|
71 |
+
|
72 |
+
if __name__ == '__main__':
|
73 |
+
app.run_server(debug=True)
|
output/top_100_update.csv
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:2ff89b933c5ee4694a0ec72fe7660677ed15012d95a0d6184767d05eb33fd397
|
3 |
+
size 16258
|
output/transactions_APE.csv
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:8c6094c453a4ae217cd7e6334ad0b92880e042d698db8b19978af61f42ceda1f
|
3 |
+
size 25981544
|
output/transactions_AXIE.csv
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:21eddd1decbb2f70f6fe9102cf81f0a4309c10d838bc9a172255ff70a2461cb8
|
3 |
+
size 7599371
|
output/transactions_GALA.csv
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:fd264d44fff732f21b170dcd839968ecb0408fba89cc6a993fa0da8f20fa8e05
|
3 |
+
size 32066355
|
output/transactions_ILV.csv
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:f5ce31d9e8d9c7b39bf1f73c2e05ac655f652ebffca121a6a662a0a89eaa62c9
|
3 |
+
size 5552703
|
output/transactions_MANA.csv
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:c8162afa63a588a222422d06a1f93508f92d225d124915c6ea51d5d051d4db1e
|
3 |
+
size 12039331
|
output/transactions_PET.csv
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:fa42451d8ab6696d44d5b648754791e1462f630de79439c580b8e92be5f016df
|
3 |
+
size 885
|
output/transactions_WEAOPON.csv
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:35e366dc8930a78bd4f37409447da3c6b7f53f3b6a699c89a4c9d9d5740622f5
|
3 |
+
size 537
|
requirements.txt
ADDED
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
beautifulsoup4
|
2 |
+
pandas
|
3 |
+
numpy
|
4 |
+
requests
|
5 |
+
lxml
|
6 |
+
dash_bootstrap_components
|
7 |
+
dash
|
8 |
+
python-dotenv
|
9 |
+
streamlit
|
10 |
+
requests
|
11 |
+
plotly
|
12 |
+
nbformat
|
scrap_data_CMC.py
ADDED
@@ -0,0 +1,88 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
#-------------------------------------libraries ----------------------------------
|
2 |
+
|
3 |
+
from requests import Request, Session
|
4 |
+
from requests.exceptions import ConnectionError, Timeout, TooManyRedirects
|
5 |
+
import json
|
6 |
+
import os
|
7 |
+
import pandas as pd
|
8 |
+
import numpy as np
|
9 |
+
import logging
|
10 |
+
from dotenv import load_dotenv
|
11 |
+
load_dotenv()
|
12 |
+
|
13 |
+
#-------------------------------------env vars----------------------------------
|
14 |
+
|
15 |
+
url = os.getenv("URL_CMC")
|
16 |
+
endpoints = ["v1/cryptocurrency/listings/latest",
|
17 |
+
"/v1/cryptocurrency/trending/latest",
|
18 |
+
]
|
19 |
+
start = "1"
|
20 |
+
stop = "100"
|
21 |
+
parameters = {
|
22 |
+
'start':start,
|
23 |
+
'limit':stop,
|
24 |
+
'convert':'USD'
|
25 |
+
}
|
26 |
+
headers = {
|
27 |
+
'Accepts': 'application/json',
|
28 |
+
'X-CMC_PRO_API_KEY': os.getenv("API_KEY_CMC"),
|
29 |
+
}
|
30 |
+
|
31 |
+
# Configure the logging settings
|
32 |
+
log_folder = "./logs/scrapping/"
|
33 |
+
os.makedirs(log_folder, exist_ok=True) # Ensure the log folder exists
|
34 |
+
log_file = os.path.join(log_folder, "scrapping.log")
|
35 |
+
log_format = "%(asctime)s [%(levelname)s] - %(message)s"
|
36 |
+
logging.basicConfig(filename=log_file, level=logging.INFO, format=log_format)
|
37 |
+
|
38 |
+
#-------------------------------------api call----------------------------------
|
39 |
+
|
40 |
+
session = Session()
|
41 |
+
session.headers.update(headers)
|
42 |
+
|
43 |
+
for endpoint in endpoints:
|
44 |
+
target = f"{url}/{endpoint}"
|
45 |
+
try:
|
46 |
+
response = session.get(target, params=parameters)
|
47 |
+
data = json.loads(response.text)
|
48 |
+
with open(f'output/cmc_data_{endpoint.replace("/", "_")}_{stop}.json', 'w') as f:
|
49 |
+
json.dump(data, f)
|
50 |
+
logging.info(f"Successfully fetched data from {target}")
|
51 |
+
except (ConnectionError, Timeout, TooManyRedirects) as e:
|
52 |
+
logging.error(f"Error while fetching data from {target}: {e}")
|
53 |
+
|
54 |
+
#-------------------------------------process data----------------------------------
|
55 |
+
|
56 |
+
# create data frame with chosen columns
|
57 |
+
df = pd.DataFrame(data["data"])[["name","symbol","circulating_supply","total_supply","quote"]]
|
58 |
+
# explode column quote then chose columns
|
59 |
+
quote_df = pd.json_normalize(df['quote'].apply(lambda x: x['USD']))[["price","percent_change_24h","percent_change_7d","percent_change_90d","market_cap","fully_diluted_market_cap","last_updated"]]
|
60 |
+
# drop quote
|
61 |
+
df = df.drop("quote",axis=1)
|
62 |
+
# create features
|
63 |
+
df["percent_tokens_circulation"] = np.round((df["circulating_supply"]/df["total_supply"])*100,1)
|
64 |
+
# merge dataframe
|
65 |
+
df = df.join(quote_df)
|
66 |
+
df["last_updated"] = pd.to_datetime(df["last_updated"])
|
67 |
+
#df.to_csv(f"output/top_{stop}_update.csv")
|
68 |
+
|
69 |
+
#-------------------------------------save data----------------------------------
|
70 |
+
|
71 |
+
# Check if the file exists
|
72 |
+
output_file = f"output/top_{stop}_update.csv"
|
73 |
+
if os.path.isfile(output_file):
|
74 |
+
logging.info("Updating dataset"+f"top_{stop}_update"+". ")
|
75 |
+
# Read the existing data
|
76 |
+
existing_data = pd.read_csv(output_file)
|
77 |
+
# Concatenate the existing data with the new data vertically
|
78 |
+
updated_data = pd.concat([existing_data, df], axis=0, ignore_index=True)
|
79 |
+
# Remove duplicates (if any) based on a unique identifier column
|
80 |
+
updated_data.drop_duplicates(subset=["symbol", "last_updated"], inplace=True)
|
81 |
+
# Save the updated data back to the same file
|
82 |
+
updated_data.to_csv(output_file, index=False)
|
83 |
+
else:
|
84 |
+
# If the file doesn't exist, save the current data to it
|
85 |
+
df.to_csv(output_file, index=False)
|
86 |
+
logging.info("Script execution completed.")
|
87 |
+
|
88 |
+
#-------------------------------------end----------------------------------
|
scrap_data_etherscan.py
ADDED
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import requests
|
2 |
+
import time
|
3 |
+
import pandas as pd
|
4 |
+
import json
|
5 |
+
import os
|
6 |
+
from utils.functions import update_and_save_csv
|
7 |
+
|
8 |
+
# Create output folder
|
9 |
+
if not os.path.exists("output"):
|
10 |
+
os.makedirs("output")
|
11 |
+
|
12 |
+
# Load the JSON file into a dictionary
|
13 |
+
print(os.getcwd())
|
14 |
+
with open("ressources/dict_tokens_addr.json", "r") as file:
|
15 |
+
dict_addresses = json.load(file)
|
16 |
+
|
17 |
+
update_and_save_csv(dict_addresses)
|