YuhangDeng123 commited on
Commit
c2c96de
1 Parent(s): 3b9bc19

Upload 3 files

Browse files
Files changed (3) hide show
  1. app.py +75 -0
  2. functions.py +91 -0
  3. requirements.txt +11 -0
app.py ADDED
@@ -0,0 +1,75 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import streamlit as st
2
+ import hopsworks
3
+ import joblib
4
+ from datetime import date
5
+ import pandas as pd
6
+ from datetime import timedelta, datetime
7
+ from functions import *
8
+ import numpy as np
9
+ from sklearn.preprocessing import StandardScaler
10
+
11
+ import folium
12
+ from streamlit_folium import st_folium, folium_static
13
+ import json
14
+ import time
15
+ from branca.element import Figure
16
+
17
+
18
+ def fancy_header(text, font_size=24):
19
+ res = f'<p style="color:#ff5f72; font-size: {font_size}px; text-align:center;">{text}</p>'
20
+ st.markdown(res, unsafe_allow_html=True)
21
+
22
+ st.set_page_config(layout="wide")
23
+
24
+ st.title('Air Quality Prediction Project🌩')
25
+
26
+ st.write(36 * "-")
27
+ fancy_header('\n Connecting to Hopsworks Feature Store...')
28
+
29
+ project = hopsworks.login()
30
+
31
+ st.write("Successfully connected!✔️")
32
+
33
+ st.write(36 * "-")
34
+ fancy_header('\n Getting data from Feature Store...')
35
+
36
+ today = date.today()
37
+ ##########################城市####################
38
+ city = "Guangzhou"
39
+ df_weather = get_weather_data_weekly(city, today)
40
+ df_weather.date = df_weather.date.apply(timestamp_2_time)
41
+ df_weather_x = df_weather.drop(columns=["date"]).fillna(0)
42
+ df_weather_nn=np.array(df_weather_x)
43
+ scaler = StandardScaler()
44
+ scaler.fit(df_weather_x)
45
+
46
+ df_weather_use=scaler.transform(df_weather_x)
47
+
48
+ df_weather_use_1= pd.DataFrame(df_weather_use)
49
+
50
+ #preds_zzz = model.predict(df_weather_use_1).astype(int)
51
+
52
+ st.write(36 * "-")
53
+
54
+ ########################根据模型名称进行修改#####################
55
+ mr = project.get_model_registry()
56
+ model = mr.get_model("air_quality_modal_choosed", version=1)
57
+ model_dir = model.download()
58
+ model = joblib.load(model_dir + "/air_quality_model_choosed.pkl")
59
+
60
+ st.write("-" * 36)
61
+
62
+
63
+ preds = model.predict(df_weather_use_1).astype(int)
64
+ pollution_level = get_aplevel(preds.T.reshape(-1, 1))
65
+
66
+ next_week = [f"{(today + timedelta(days=d)).strftime('%Y-%m-%d')},{(today + timedelta(days=d)).strftime('%A')}" for d in range(8)]
67
+
68
+ df = pd.DataFrame(data=[preds, pollution_level], index=["AQI", "Air pollution level"], columns=next_week)
69
+ ###########如果报错AQI这个修改成preds的标签##################
70
+
71
+ st.write(df)
72
+
73
+ st.button("Re-run")
74
+
75
+
functions.py ADDED
@@ -0,0 +1,91 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import requests
2
+ import os
3
+ import joblib
4
+ import pandas as pd
5
+ import datetime
6
+ import numpy as np
7
+ import time
8
+ from sklearn.preprocessing import OrdinalEncoder
9
+ from dotenv import load_dotenv
10
+ load_dotenv(override=True)
11
+
12
+
13
+ def decode_features(df, feature_view):
14
+ """Decodes features in the input DataFrame using corresponding Hopsworks Feature Store transformation functions"""
15
+ df_res = df.copy()
16
+
17
+ import inspect
18
+
19
+
20
+ td_transformation_functions = feature_view._batch_scoring_server._transformation_functions
21
+
22
+ res = {}
23
+ for feature_name in td_transformation_functions:
24
+ if feature_name in df_res.columns:
25
+ td_transformation_function = td_transformation_functions[feature_name]
26
+ sig, foobar_locals = inspect.signature(td_transformation_function.transformation_fn), locals()
27
+ param_dict = dict([(param.name, param.default) for param in sig.parameters.values() if param.default != inspect._empty])
28
+ if td_transformation_function.name == "min_max_scaler":
29
+ df_res[feature_name] = df_res[feature_name].map(
30
+ lambda x: x * (param_dict["max_value"] - param_dict["min_value"]) + param_dict["min_value"])
31
+
32
+ elif td_transformation_function.name == "standard_scaler":
33
+ df_res[feature_name] = df_res[feature_name].map(
34
+ lambda x: x * param_dict['std_dev'] + param_dict["mean"])
35
+ elif td_transformation_function.name == "label_encoder":
36
+ dictionary = param_dict['value_to_index']
37
+ dictionary_ = {v: k for k, v in dictionary.items()}
38
+ df_res[feature_name] = df_res[feature_name].map(
39
+ lambda x: dictionary_[x])
40
+ return df_res
41
+
42
+
43
+ def get_model(project, model_name, evaluation_metric, sort_metrics_by):
44
+ """Retrieve desired model or download it from the Hopsworks Model Registry.
45
+ In second case, it will be physically downloaded to this directory"""
46
+ TARGET_FILE = "model.pkl"
47
+ list_of_files = [os.path.join(dirpath,filename) for dirpath, _, filenames \
48
+ in os.walk('.') for filename in filenames if filename == TARGET_FILE]
49
+
50
+ if list_of_files:
51
+ model_path = list_of_files[0]
52
+ model = joblib.load(model_path)
53
+ else:
54
+ if not os.path.exists(TARGET_FILE):
55
+ mr = project.get_model_registry()
56
+ # get best model based on custom metrics
57
+ model = mr.get_best_model(model_name,
58
+ evaluation_metric,
59
+ sort_metrics_by)
60
+ model_dir = model.download()
61
+ model = joblib.load(model_dir + "/model.pkl")
62
+
63
+ return model
64
+
65
+
66
+ def get_weather_data_weekly(city: str, start_date: datetime) -> pd.DataFrame:
67
+ next7days_weather=pd.read_csv('https://weather.visualcrossing.com/VisualCrossingWebServices/rest/services/timeline/Guangzhou/next7days?unitGroup=metric&include=days&key=5WNL2M94KKQ4R4F32LFV8DPE4&contentType=csv')
68
+ ########################城市名############################
69
+
70
+ df_weather = pd.DataFrame(next7days_weather)
71
+ df_weather.rename(columns = {"datetime": "date"},
72
+ inplace = True)
73
+ #########################根据模型的feature进行修改###############################
74
+ df_weather = df_weather.drop(labels=['stations','icon','description','conditions','sunset','sunrise','severerisk','preciptype','name','feelslikemax','temp','precipprob','windspeed','cloudcover','precip','tempmax','uvindex','solarradiation','solarenergy','winddir','moonphase','snow','snowdepth'], axis=1)
75
+
76
+ return df_weather
77
+
78
+ def get_aplevel(temps:np.ndarray) -> list:
79
+ boundary_list = np.array([0, 50, 100, 150, 200, 300]) # assert temps.shape == [x, 1]
80
+ redf = np.logical_not(temps<=boundary_list) # temps.shape[0] x boundary_list.shape[0] ndarray
81
+ hift = np.concatenate((np.roll(redf, -1)[:, :-1], np.full((temps.shape[0], 1), False)), axis = 1)
82
+ cat = np.nonzero(np.not_equal(redf,hift))
83
+
84
+ air_pollution_level = ['Good', 'Moderate', 'Unhealthy for sensitive Groups','Unhealthy' ,'Very Unhealthy', 'Hazardous']
85
+ level = [air_pollution_level[el] for el in cat[1]]
86
+ return level
87
+
88
+ def timestamp_2_time(x):
89
+ dt_obj = datetime.datetime.strptime(str(x), '%Y-%m-%d')
90
+ dt_obj = dt_obj.timestamp() * 1000
91
+ return int(dt_obj)
requirements.txt ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ hopsworks
2
+ scikit-learn == 1.0.2
3
+ numpy
4
+ joblib
5
+ python-dotenv
6
+ xgboost == 0.90
7
+ optuna
8
+ datetime
9
+ folium
10
+ streamlit_folium
11
+ streamlit