created last year features; past_data_api_call.py; modified data_loading.py

dataset.csv

@@ -5,5 +5,5 @@ date,NO2,O3,wind_speed,mean_temp,global_radiation,percipitation,pressure,minimum
 2024-10-19,24.532038834951457,23.604722719141325,43,147,43,28,10140,236,92,Saturday
 2024-10-20,23.019101941747575,24.173377192982453,68,145,0,0,10160,241,82,Sunday
 2024-10-21,21.275629139072848,25.05873563218391,58,144,27,43,10206,220,92,Monday
-2024-10-22,22.334374999999998,24.5942194092827,76,123,60,12,10265,100,87,Tuesday
-2024-10-23,24.261733333333336,23.56,31,115,7,0,10328,112,95,Wednesday
+2024-10-22,22.334374999999998,24.5942194092827,76,123,57,12,10265,100,87,Tuesday
+2024-10-23,24.261733333333336,23.56,31,115,7,0,10328,105,95,Wednesday

past_data_api_calls.py

@@ -0,0 +1,209 @@
+import codecs
+import csv
+import http.client
+import os
+import re
+import sys
+import urllib.request
+from datetime import date, timedelta
+from io import StringIO
+
+import pandas as pd
+
+
+def pollution_data():
+    particles = ["NO2", "O3"]
+    stations = ["NL10636", "NL10639", "NL10643"]
+    last_year_date = date.today() - timedelta(days=365)
+    start_date = last_year_date - timedelta(days=7)
+    end_date = last_year_date + timedelta(days=3)
+    date_list = [start_date + timedelta(days=x) for x in range((end_date - start_date).days + 1)]
+    for current_date in date_list:
+        today = current_date.isoformat() + "T09:00:00Z"
+        yesterday = (current_date - timedelta(1)).isoformat() + "T09:00:00Z"
+        for particle in particles:
+            all_dataframes = []  # Reset for each particle
+            for station in stations:
+                conn = http.client.HTTPSConnection("api.luchtmeetnet.nl")
+                payload = ''
+                headers = {}
+                conn.request("GET", f"/open_api/measurements?station_number={station}&formula={particle}&page=1&order_by=timestamp_measured&order_direction=desc&end={today}&start={yesterday}", payload, headers)
+                res = conn.getresponse()
+                data = res.read()
+                decoded_data = data.decode("utf-8")
+                df = pd.read_csv(StringIO(decoded_data))
+                df = df.filter(like='value')
+                all_dataframes.append(df)
+            if all_dataframes:
+                combined_data = pd.concat(all_dataframes, ignore_index=True)
+                combined_data.to_csv(f'{particle}_{today}.csv', index=False)
+
+def delete_csv(csvs):
+    for csv_file in csvs:
+        if(os.path.exists(csv_file) and os.path.isfile(csv_file)): 
+            os.remove(csv_file)
+
+def clean_values():
+    particles = ["NO2", "O3"]
+    csvs = []
+    NO2 = []
+    O3 = []
+    last_year_date = date.today() - timedelta(days=365)
+    start_date = last_year_date - timedelta(days=7)
+    end_date = last_year_date + timedelta(days=3)
+    date_list = [start_date + timedelta(days=x) for x in range((end_date - start_date).days + 1)]
+    for current_date in date_list:
+        today = current_date.isoformat() + "T09:00:00Z"
+        for particle in particles:
+            name = f'{particle}_{today}.csv'
+            csvs.append(name)
+    for csv_file in csvs:
+        if not os.path.exists(csv_file):
+            continue  # Skip if the file doesn't exist
+        values = []  # Reset values for each CSV file
+        # Open the CSV file and read the values
+        with open(csv_file, 'r') as file:
+            reader = csv.reader(file)
+            for row in reader:
+                for value in row:
+                    # Use regular expressions to extract numeric part
+                    cleaned_value = re.findall(r"[-+]?\d*\.\d+|\d+", value)
+                    if cleaned_value:  # If we successfully extract a number
+                        values.append(float(cleaned_value[0]))  # Convert the first match to float
+
+        # Compute the average if the values list is not empty
+        if values:
+            avg = sum(values) / len(values)
+            if "NO2" in csv_file:
+                NO2.append(avg)
+            else:
+                O3.append(avg)
+    delete_csv(csvs)
+    return NO2, O3
+
+def add_columns():
+    file_path = 'weather_data.csv'
+    df = pd.read_csv(file_path)
+
+    df.insert(1, 'NO2', None)
+    df.insert(2, 'O3', None)
+    df.insert(10, 'weekday', None)
+
+    return df
+
+def scale(data):
+    df = data
+    columns = list(df.columns)
+
+    columns.insert(3, columns.pop(6))
+    df = df[columns]
+
+    columns.insert(5, columns.pop(9))
+    df = df[columns]
+
+    columns.insert(9, columns.pop(6))
+    df = df[columns]
+
+    df = df.rename(columns={
+        'datetime':'date',
+        'windspeed': 'wind_speed',
+        'temp': 'mean_temp',
+        'solarradiation':'global_radiation',
+        'precip':'percipitation',
+        'sealevelpressure':'pressure',
+        'visibility':'minimum_visibility'
+    })
+
+    df['date'] = pd.to_datetime(df['date'])
+    df['weekday'] = df['date'].dt.day_name()
+
+    df = df.sort_values(by='date').reset_index(drop=True)
+
+    df['wind_speed'] = (df['wind_speed'] / 3.6) * 10
+    df['mean_temp'] = df['mean_temp'] * 10
+    df['minimum_visibility'] = df['minimum_visibility'] * 10
+    df['percipitation'] = df['percipitation'] * 10
+    df['pressure'] = df['pressure']
+
+    df['wind_speed'] = df['wind_speed'].astype(int)
+    df['mean_temp'] = df['mean_temp'].astype(int)
+    df['minimum_visibility'] = df['minimum_visibility'].astype(int)
+    df['percipitation'] = df['percipitation'].astype(int)
+    df['pressure'] = df['pressure'].astype(int)
+    df['humidity'] = df['humidity'].astype(int)
+    df['global_radiation'] = df['global_radiation'].astype(int)
+
+    return df
+
+def insert_pollution(NO2, O3, data):
+    df = data
+    df['NO2'] = NO2
+    df['O3'] = O3
+    return df
+
+def weather_data():
+    # Get last year's same day
+    last_year_date = date.today() - timedelta(days=365)
+    # Start date is 7 days prior
+    start_date = (last_year_date - timedelta(days=7)).isoformat()
+    # End date is 3 days ahead
+    end_date = (last_year_date + timedelta(days=3)).isoformat()
+    try: 
+        ResultBytes = urllib.request.urlopen(f"https://weather.visualcrossing.com/VisualCrossingWebServices/rest/services/timeline/Utrecht/{start_date}/{end_date}?unitGroup=metric&elements=datetime%2Cwindspeed%2Ctemp%2Csolarradiation%2Cprecip%2Cpressure%2Cvisibility%2Chumidity&include=days&key=7Y6AY56M6RWVNHQ3SAVHNJWFS&maxStations=1&contentType=csv")
+        
+        # Parse the results as CSV
+        CSVText = csv.reader(codecs.iterdecode(ResultBytes, 'utf-8'))
+        # Saving the CSV content to a file
+        current_dir = os.path.dirname(os.path.realpath(__file__))
+        file_path = os.path.join(current_dir, 'weather_data.csv')
+        with open(file_path, 'w', newline='', encoding='utf-8') as csvfile:
+              csv_writer = csv.writer(csvfile)
+              csv_writer.writerows(CSVText)
+    
+    except urllib.error.HTTPError  as e:
+        ErrorInfo= e.read().decode() 
+        print('Error code: ', e.code, ErrorInfo)
+        sys.exit()
+    except  urllib.error.URLError as e:
+        ErrorInfo= e.read().decode() 
+        print('Error code: ', e.code,ErrorInfo)
+        sys.exit()
+
+def weather_data():
+    # Set up dates for last year: 7 days before today last year, and 3 days ahead of this day last year
+    today_last_year = date.today() - timedelta(365)
+    start_last_year = today_last_year - timedelta(8)
+    end_last_year = today_last_year + timedelta(2)
+    
+    try: 
+        # API call with new date range for last year
+        ResultBytes = urllib.request.urlopen(f"https://weather.visualcrossing.com/VisualCrossingWebServices/rest/services/timeline/Utrecht/{start_last_year}/{end_last_year}?unitGroup=metric&elements=datetime%2Cwindspeed%2Ctemp%2Csolarradiation%2Cprecip%2Cpressure%2Cvisibility%2Chumidity&include=days&key=7Y6AY56M6RWVNHQ3SAVHNJWFS&maxStations=1&contentType=csv")
+        
+        # Parse the results as CSV
+        CSVText = csv.reader(codecs.iterdecode(ResultBytes, 'utf-8'))
+        # Saving the CSV content to a file
+        current_dir = os.path.dirname(os.path.realpath(__file__))
+        file_path = os.path.join(current_dir, 'weather_data.csv')
+        with open(file_path, 'w', newline='', encoding='utf-8') as csvfile:
+            csv_writer = csv.writer(csvfile)
+            csv_writer.writerows(CSVText)
+    
+    except urllib.error.HTTPError as e:
+        ErrorInfo = e.read().decode() 
+        print('Error code: ', e.code, ErrorInfo)
+        sys.exit()
+    except urllib.error.URLError as e:
+        ErrorInfo = e.read().decode() 
+        print('Error code: ', e.code, ErrorInfo)
+        sys.exit()
+
+
+def get_past_data():
+    weather_data()
+    pollution_data()
+    NO2, O3 = clean_values()
+    df = add_columns()
+    scaled_df = scale(df)
+    output_df = insert_pollution(NO2, O3, scaled_df)
+    os.remove('weather_data.csv')
+    return output_df

src/data_loading.py

@@ -1,6 +1,9 @@
+import joblib
 import numpy as np
 import pandas as pd
-import joblib
+
+from past_data_api_calls import get_past_data
+
 
 def create_features(
     data,
@@ -83,19 +86,21 @@ def create_features(
             data[feature].rolling(window=sma_days).mean()
         )
 
+    past_data = get_past_data()
     # Create particle data (NO2 and O3) from the same time last year
     # Today last year
-    data["O3_last_year"] = 0 # data["O3_last_year"] = data["O3"].shift(365)
-    data["NO2_last_year"] = 0 # data["NO2_last_year"] = data["NO2"].shift(365)
+
+    data["O3_last_year"] = past_data["O3"].iloc[-4] # data["O3_last_year"] = data["O3"].shift(365)
+    data["NO2_last_year"] = past_data["NO2"].iloc[-4] # data["NO2_last_year"] = data["NO2"].shift(365)
 
     # 7 days before today last year
-    for i in range(1, lag_days + 1):
-        data[f"O3_last_year_{i}_days_before"] = 0 # data["O3"].shift(365 + i)
-        data[f"NO2_last_year_{i}_days_before"] = 0 # data["NO2"].shift(365 + i)
+    for i in range(1, lag_days+1):
+        data[f"O3_last_year_{i}_days_before"] = past_data["O3"].iloc[i-1] # data["O3"].shift(365 + i)
+        data[f"NO2_last_year_{i}_days_before"] = past_data["NO2"].iloc[i-1] # data["NO2"].shift(365 + i)
 
     # 3 days after today last year
-    data["O3_last_year_3_days_after"] = 0 # data["O3"].shift(365 - 3)
-    data["NO2_last_year_3_days_after"] = 0 # data["NO2"].shift(365 - 3)
+    data["O3_last_year_3_days_after"] = past_data["O3"].iloc[-1] # data["O3"].shift(365 - 3)
+    data["NO2_last_year_3_days_after"] = past_data["NO2"].iloc[-1] # data["NO2"].shift(365 - 3)
 
     # Calculate the number of rows before dropping missing values
     rows_before = data.shape[0]