Spaces:
Runtime error
Runtime error
Commit
•
c2c96de
1
Parent(s):
3b9bc19
Upload 3 files
Browse files- app.py +75 -0
- functions.py +91 -0
- requirements.txt +11 -0
app.py
ADDED
@@ -0,0 +1,75 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import streamlit as st
|
2 |
+
import hopsworks
|
3 |
+
import joblib
|
4 |
+
from datetime import date
|
5 |
+
import pandas as pd
|
6 |
+
from datetime import timedelta, datetime
|
7 |
+
from functions import *
|
8 |
+
import numpy as np
|
9 |
+
from sklearn.preprocessing import StandardScaler
|
10 |
+
|
11 |
+
import folium
|
12 |
+
from streamlit_folium import st_folium, folium_static
|
13 |
+
import json
|
14 |
+
import time
|
15 |
+
from branca.element import Figure
|
16 |
+
|
17 |
+
|
18 |
+
def fancy_header(text, font_size=24):
|
19 |
+
res = f'<p style="color:#ff5f72; font-size: {font_size}px; text-align:center;">{text}</p>'
|
20 |
+
st.markdown(res, unsafe_allow_html=True)
|
21 |
+
|
22 |
+
st.set_page_config(layout="wide")
|
23 |
+
|
24 |
+
st.title('Air Quality Prediction Project🌩')
|
25 |
+
|
26 |
+
st.write(36 * "-")
|
27 |
+
fancy_header('\n Connecting to Hopsworks Feature Store...')
|
28 |
+
|
29 |
+
project = hopsworks.login()
|
30 |
+
|
31 |
+
st.write("Successfully connected!✔️")
|
32 |
+
|
33 |
+
st.write(36 * "-")
|
34 |
+
fancy_header('\n Getting data from Feature Store...')
|
35 |
+
|
36 |
+
today = date.today()
|
37 |
+
##########################城市####################
|
38 |
+
city = "Guangzhou"
|
39 |
+
df_weather = get_weather_data_weekly(city, today)
|
40 |
+
df_weather.date = df_weather.date.apply(timestamp_2_time)
|
41 |
+
df_weather_x = df_weather.drop(columns=["date"]).fillna(0)
|
42 |
+
df_weather_nn=np.array(df_weather_x)
|
43 |
+
scaler = StandardScaler()
|
44 |
+
scaler.fit(df_weather_x)
|
45 |
+
|
46 |
+
df_weather_use=scaler.transform(df_weather_x)
|
47 |
+
|
48 |
+
df_weather_use_1= pd.DataFrame(df_weather_use)
|
49 |
+
|
50 |
+
#preds_zzz = model.predict(df_weather_use_1).astype(int)
|
51 |
+
|
52 |
+
st.write(36 * "-")
|
53 |
+
|
54 |
+
########################根据模型名称进行修改#####################
|
55 |
+
mr = project.get_model_registry()
|
56 |
+
model = mr.get_model("air_quality_modal_choosed", version=1)
|
57 |
+
model_dir = model.download()
|
58 |
+
model = joblib.load(model_dir + "/air_quality_model_choosed.pkl")
|
59 |
+
|
60 |
+
st.write("-" * 36)
|
61 |
+
|
62 |
+
|
63 |
+
preds = model.predict(df_weather_use_1).astype(int)
|
64 |
+
pollution_level = get_aplevel(preds.T.reshape(-1, 1))
|
65 |
+
|
66 |
+
next_week = [f"{(today + timedelta(days=d)).strftime('%Y-%m-%d')},{(today + timedelta(days=d)).strftime('%A')}" for d in range(8)]
|
67 |
+
|
68 |
+
df = pd.DataFrame(data=[preds, pollution_level], index=["AQI", "Air pollution level"], columns=next_week)
|
69 |
+
###########如果报错AQI这个修改成preds的标签##################
|
70 |
+
|
71 |
+
st.write(df)
|
72 |
+
|
73 |
+
st.button("Re-run")
|
74 |
+
|
75 |
+
|
functions.py
ADDED
@@ -0,0 +1,91 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import requests
|
2 |
+
import os
|
3 |
+
import joblib
|
4 |
+
import pandas as pd
|
5 |
+
import datetime
|
6 |
+
import numpy as np
|
7 |
+
import time
|
8 |
+
from sklearn.preprocessing import OrdinalEncoder
|
9 |
+
from dotenv import load_dotenv
|
10 |
+
load_dotenv(override=True)
|
11 |
+
|
12 |
+
|
13 |
+
def decode_features(df, feature_view):
|
14 |
+
"""Decodes features in the input DataFrame using corresponding Hopsworks Feature Store transformation functions"""
|
15 |
+
df_res = df.copy()
|
16 |
+
|
17 |
+
import inspect
|
18 |
+
|
19 |
+
|
20 |
+
td_transformation_functions = feature_view._batch_scoring_server._transformation_functions
|
21 |
+
|
22 |
+
res = {}
|
23 |
+
for feature_name in td_transformation_functions:
|
24 |
+
if feature_name in df_res.columns:
|
25 |
+
td_transformation_function = td_transformation_functions[feature_name]
|
26 |
+
sig, foobar_locals = inspect.signature(td_transformation_function.transformation_fn), locals()
|
27 |
+
param_dict = dict([(param.name, param.default) for param in sig.parameters.values() if param.default != inspect._empty])
|
28 |
+
if td_transformation_function.name == "min_max_scaler":
|
29 |
+
df_res[feature_name] = df_res[feature_name].map(
|
30 |
+
lambda x: x * (param_dict["max_value"] - param_dict["min_value"]) + param_dict["min_value"])
|
31 |
+
|
32 |
+
elif td_transformation_function.name == "standard_scaler":
|
33 |
+
df_res[feature_name] = df_res[feature_name].map(
|
34 |
+
lambda x: x * param_dict['std_dev'] + param_dict["mean"])
|
35 |
+
elif td_transformation_function.name == "label_encoder":
|
36 |
+
dictionary = param_dict['value_to_index']
|
37 |
+
dictionary_ = {v: k for k, v in dictionary.items()}
|
38 |
+
df_res[feature_name] = df_res[feature_name].map(
|
39 |
+
lambda x: dictionary_[x])
|
40 |
+
return df_res
|
41 |
+
|
42 |
+
|
43 |
+
def get_model(project, model_name, evaluation_metric, sort_metrics_by):
|
44 |
+
"""Retrieve desired model or download it from the Hopsworks Model Registry.
|
45 |
+
In second case, it will be physically downloaded to this directory"""
|
46 |
+
TARGET_FILE = "model.pkl"
|
47 |
+
list_of_files = [os.path.join(dirpath,filename) for dirpath, _, filenames \
|
48 |
+
in os.walk('.') for filename in filenames if filename == TARGET_FILE]
|
49 |
+
|
50 |
+
if list_of_files:
|
51 |
+
model_path = list_of_files[0]
|
52 |
+
model = joblib.load(model_path)
|
53 |
+
else:
|
54 |
+
if not os.path.exists(TARGET_FILE):
|
55 |
+
mr = project.get_model_registry()
|
56 |
+
# get best model based on custom metrics
|
57 |
+
model = mr.get_best_model(model_name,
|
58 |
+
evaluation_metric,
|
59 |
+
sort_metrics_by)
|
60 |
+
model_dir = model.download()
|
61 |
+
model = joblib.load(model_dir + "/model.pkl")
|
62 |
+
|
63 |
+
return model
|
64 |
+
|
65 |
+
|
66 |
+
def get_weather_data_weekly(city: str, start_date: datetime) -> pd.DataFrame:
|
67 |
+
next7days_weather=pd.read_csv('https://weather.visualcrossing.com/VisualCrossingWebServices/rest/services/timeline/Guangzhou/next7days?unitGroup=metric&include=days&key=5WNL2M94KKQ4R4F32LFV8DPE4&contentType=csv')
|
68 |
+
########################城市名############################
|
69 |
+
|
70 |
+
df_weather = pd.DataFrame(next7days_weather)
|
71 |
+
df_weather.rename(columns = {"datetime": "date"},
|
72 |
+
inplace = True)
|
73 |
+
#########################根据模型的feature进行修改###############################
|
74 |
+
df_weather = df_weather.drop(labels=['stations','icon','description','conditions','sunset','sunrise','severerisk','preciptype','name','feelslikemax','temp','precipprob','windspeed','cloudcover','precip','tempmax','uvindex','solarradiation','solarenergy','winddir','moonphase','snow','snowdepth'], axis=1)
|
75 |
+
|
76 |
+
return df_weather
|
77 |
+
|
78 |
+
def get_aplevel(temps:np.ndarray) -> list:
|
79 |
+
boundary_list = np.array([0, 50, 100, 150, 200, 300]) # assert temps.shape == [x, 1]
|
80 |
+
redf = np.logical_not(temps<=boundary_list) # temps.shape[0] x boundary_list.shape[0] ndarray
|
81 |
+
hift = np.concatenate((np.roll(redf, -1)[:, :-1], np.full((temps.shape[0], 1), False)), axis = 1)
|
82 |
+
cat = np.nonzero(np.not_equal(redf,hift))
|
83 |
+
|
84 |
+
air_pollution_level = ['Good', 'Moderate', 'Unhealthy for sensitive Groups','Unhealthy' ,'Very Unhealthy', 'Hazardous']
|
85 |
+
level = [air_pollution_level[el] for el in cat[1]]
|
86 |
+
return level
|
87 |
+
|
88 |
+
def timestamp_2_time(x):
|
89 |
+
dt_obj = datetime.datetime.strptime(str(x), '%Y-%m-%d')
|
90 |
+
dt_obj = dt_obj.timestamp() * 1000
|
91 |
+
return int(dt_obj)
|
requirements.txt
ADDED
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
hopsworks
|
2 |
+
scikit-learn == 1.0.2
|
3 |
+
numpy
|
4 |
+
joblib
|
5 |
+
python-dotenv
|
6 |
+
xgboost == 0.90
|
7 |
+
optuna
|
8 |
+
datetime
|
9 |
+
folium
|
10 |
+
streamlit_folium
|
11 |
+
streamlit
|