File size: 5,092 Bytes
6a440fc 5064f83 6a440fc 5064f83 6a440fc 2c18c58 6a440fc 2c18c58 6a440fc 2c18c58 6a440fc eeaf86d 6a440fc eeaf86d 6a440fc 5064f83 6a440fc eeaf86d 6a440fc eeaf86d 6a440fc 5064f83 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 |
import numpy as np
import pandas as pd
import joblib
def create_features(
data,
target_particle, # Added this parameter
lag_days=7,
sma_days=7,
):
"""
Creates lagged features, SMA features, last year's particle data (NO2 and O3) for specific days,
sine and cosine transformations for 'weekday' and 'month', and target variables for the specified
particle ('O3' or 'NO2') for the next 'days_ahead' days. Scales features and targets without
disregarding outliers and saves the scalers for inverse scaling. Splits the data into train,
validation, and test sets using the most recent dates. Prints the number of rows with missing
values dropped from the dataset.
Parameters:
- data (pd.DataFrame): The input time-series dataset.
- target_particle (str): The target particle ('O3' or 'NO2') for which targets are created.
- lag_days (int): Number of lag days to create features for (default 7).
- sma_days (int): Window size for Simple Moving Average (default 7).
- days_ahead (int): Number of days ahead to create target variables for (default 3).
Returns:
- X_train_scaled (pd.DataFrame): Scaled training features.
- y_train_scaled (pd.DataFrame): Scaled training targets.
- X_val_scaled (pd.DataFrame): Scaled validation features (365 days).
- y_val_scaled (pd.DataFrame): Scaled validation targets (365 days).
- X_test_scaled (pd.DataFrame): Scaled test features (365 days).
- y_test_scaled (pd.DataFrame): Scaled test targets (365 days).
"""
import warnings
import numpy as np
import pandas as pd
from sklearn.preprocessing import StandardScaler
warnings.filterwarnings("ignore")
lag_features = [
"NO2",
"O3",
"wind_speed",
"mean_temp",
"global_radiation",
"minimum_visibility",
"humidity",
]
if target_particle == "NO2":
lag_features = lag_features + ["percipitation", "pressure"]
if target_particle not in ["O3", "NO2"]:
raise ValueError("target_particle must be 'O3' or 'NO2'")
data = data.copy()
data["date"] = pd.to_datetime(data["date"])
data = data.sort_values("date").reset_index(drop=True)
# Extract 'weekday' and 'month' from 'date' if not present
if "weekday" not in data.columns or data["weekday"].dtype == object:
data["weekday"] = data["date"].dt.weekday # Monday=0, Sunday=6
if "month" not in data.columns:
data["month"] = data["date"].dt.month # 1 to 12
# Create sine and cosine transformations for 'weekday' and 'month'
data["weekday_sin"] = np.sin(2 * np.pi * data["weekday"] / 7)
data["weekday_cos"] = np.cos(2 * np.pi * data["weekday"] / 7)
data["month_sin"] = np.sin(
2 * np.pi * (data["month"] - 1) / 12
) # Adjust month to 0-11
data["month_cos"] = np.cos(2 * np.pi * (data["month"] - 1) / 12)
# Create lagged features for the specified lag days
for feature in lag_features:
for lag in range(1, lag_days + 1):
data[f"{feature}_lag_{lag}"] = data[feature].shift(lag)
# Create SMA features
for feature in lag_features:
data[f"{feature}_sma_{sma_days}"] = (
data[feature].rolling(window=sma_days).mean()
)
# Create particle data (NO2 and O3) from the same time last year
# Today last year
data["O3_last_year"] = 0 # data["O3_last_year"] = data["O3"].shift(365)
data["NO2_last_year"] = 0 # data["NO2_last_year"] = data["NO2"].shift(365)
# 7 days before today last year
for i in range(1, lag_days + 1):
data[f"O3_last_year_{i}_days_before"] = 0 # data["O3"].shift(365 + i)
data[f"NO2_last_year_{i}_days_before"] = 0 # data["NO2"].shift(365 + i)
# 3 days after today last year
data["O3_last_year_3_days_after"] = 0 # data["O3"].shift(365 - 3)
data["NO2_last_year_3_days_after"] = 0 # data["NO2"].shift(365 - 3)
# Calculate the number of rows before dropping missing values
rows_before = data.shape[0]
# Drop missing values
data = data.dropna().reset_index(drop=True)
# Calculate the number of rows after dropping missing values
rows_after = data.shape[0]
# Calculate and print the number of rows dropped
rows_dropped = rows_before - rows_after
print(f"Number of rows with missing values dropped: {rows_dropped}")
# Ensure the data is sorted by date in ascending order
data = data.sort_values("date").reset_index(drop=True)
# Define feature columns
exclude_cols = ["date", "weekday", "month"]
feature_cols = [col for col in data.columns if col not in exclude_cols]
# Split features and targets
x = data[feature_cols]
# Initialize scalers
feature_scaler = joblib.load(f"scalers/feature_scaler_{target_particle}.joblib")
# Fit the scalers on the training data
X_scaled = feature_scaler.fit_transform(x)
# Convert scaled data back to DataFrame for consistency
X_scaled = pd.DataFrame(
X_scaled, columns=feature_cols, index=x.index
)
return X_scaled
|