bigger changes

.gitattributes

@@ -1,35 +0,0 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text

.github/workflows/air-quality-daily.yml

@@ -1,39 +0,0 @@
-# name: air-quality-daily
-
-# on:
-#   schedule:
-#     - cron: "*/5 * * * *" # Run every 5 minutes (minimum allowed)
-
-# jobs:
-#   schedule_pipelines:
-#     runs-on: ubuntu-latest
-
-#     permissions:
-#       pages: write
-#       contents: write
-
-#     steps:
-#       - name: checkout repo content
-#         uses: actions/checkout@v4
-
-#       - name: Update dummy file
-#         run: echo "$(date)" >> dummy.txt
-     
-
-#       - name: pushing changes
-#         uses: stefanzweifel/git-auto-commit-action@v5
-#         with:
-#           commit_message: "Updating dashboard"
-#           commit_options: '--no-verify --signoff'
-
-#           repository: .
-
-#           status_options: '--untracked-files=no'
-
-#           skip_dirty_check: true
-
-#           skip_fetch: true
-
-#           skip_checkout: true
-
-#           push_options: '--force'

.github/workflows/huggingface-sync.yml

@@ -9,7 +9,7 @@ jobs:
   sync-to-hub:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
         with:
           fetch-depth: 0
           lfs: true

.gitignore

@@ -3,3 +3,4 @@
 .env
 .cache.sqlite
 __pycache__/
+debug.ipynb

app_streamlit.py

@@ -9,8 +9,6 @@ import pickle
 import plotly.express as px
 import json
 from datetime import datetime
-from apscheduler.schedulers.background import BackgroundScheduler
-from huggingface_hub import restart_space
 import os
 
 

debug.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [
     {
@@ -55,7 +55,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [
     {

dummy.txt

@@ -1,28 +0,0 @@
-Wed Nov 20 12:42:00 UTC 2024
-Wed Nov 20 13:24:46 UTC 2024
-Wed Nov 20 13:40:30 UTC 2024
-Wed Nov 20 13:47:42 UTC 2024
-Wed Nov 20 13:54:50 UTC 2024
-Wed Nov 20 14:09:39 UTC 2024
-Wed Nov 20 14:30:34 UTC 2024
-Wed Nov 20 14:43:23 UTC 2024
-Wed Nov 20 14:50:17 UTC 2024
-Wed Nov 20 14:57:42 UTC 2024
-Wed Nov 20 15:16:57 UTC 2024
-Wed Nov 20 15:33:10 UTC 2024
-Wed Nov 20 15:44:06 UTC 2024
-Wed Nov 20 15:51:16 UTC 2024
-Wed Nov 20 15:58:29 UTC 2024
-Wed Nov 20 16:22:33 UTC 2024
-Wed Nov 20 16:43:26 UTC 2024
-Wed Nov 20 16:54:17 UTC 2024
-Wed Nov 20 17:08:24 UTC 2024
-Wed Nov 20 17:28:09 UTC 2024
-Wed Nov 20 17:38:56 UTC 2024
-Wed Nov 20 17:46:05 UTC 2024
-Wed Nov 20 17:53:24 UTC 2024
-Wed Nov 20 18:09:58 UTC 2024
-Wed Nov 20 18:36:12 UTC 2024
-Wed Nov 20 18:49:16 UTC 2024
-Wed Nov 20 18:56:21 UTC 2024
-Wed Nov 20 19:10:45 UTC 2024

functions/figure.py

@@ -109,7 +109,7 @@ def plot(df, n=10):
         ),
         autosize=True,
         width=2100,
-        height=900,
+        height=750,
         hoverlabel=dict(
             font_size=20  # Increase hover label font size
         )

infer.py

@@ -1,102 +0,0 @@
-import datetime
-import pandas as pd
-from xgboost import XGBRegressor
-import hopsworks
-import json
-from functions import util
-import os
-
-# Set up
-
-api_key = os.getenv('HOPSWORKS_API_KEY')
-project_name = os.getenv('HOPSWORKS_PROJECT')
-
-project = hopsworks.login(project=project_name, api_key_value=api_key)
-fs = project.get_feature_store() 
-secrets = util.secrets_api(project.name)
-location_str = secrets.get_secret("SENSOR_LOCATION_JSON").value
-location = json.loads(location_str)
-country=location['country']
-city=location['city']
-street=location['street']
-
-AQI_API_KEY = secrets.get_secret("AQI_API_KEY").value
-location_str = secrets.get_secret("SENSOR_LOCATION_JSON").value
-location = json.loads(location_str)
-
-today = datetime.datetime.now() - datetime.timedelta(0)
-
-feature_view = fs.get_feature_view(
-    name='air_quality_fv',
-    version=1,
-)
-
-### Retreive model
-
-mr = project.get_model_registry()
-
-retrieved_model = mr.get_model(
-    name="air_quality_xgboost_model",
-    version=1,
-)
-
-saved_model_dir = retrieved_model.download()
-retrieved_xgboost_model = XGBRegressor()
-retrieved_xgboost_model.load_model(saved_model_dir + "/model.json")
-
-### Retrieve features 
-
-weather_fg = fs.get_feature_group(
-    name='weather',
-    version=1,
-)
-
-today_timestamp = pd.to_datetime(today)
-batch_data = weather_fg.filter(weather_fg.date >= today_timestamp ).read()
-
-
-### Predict and upload
-
-batch_data['predicted_pm25'] = retrieved_xgboost_model.predict(
-    batch_data[['temperature_2m_mean', 'precipitation_sum', 'wind_speed_10m_max', 'wind_direction_10m_dominant']])
-
-batch_data['street'] = street
-batch_data['city'] = city
-batch_data['country'] = country
-# Fill in the number of days before the date on which you made the forecast (base_date)
-batch_data['days_before_forecast_day'] = range(1, len(batch_data)+1)
-batch_data = batch_data.sort_values(by=['date'])
-#batch_data['date'] = batch_data['date'].dt.tz_convert(None).astype('datetime64[ns]')
-
-plt = util.plot_air_quality_forecast(city, street, batch_data, file_path="./img/pm25_forecast.png")
-
-monitor_fg = fs.get_or_create_feature_group(
-    name='aq_predictions',
-    description='Air Quality prediction monitoring',
-    version=1,
-    primary_key=['city','street','date','days_before_forecast_day'],
-    event_time="date"
-)
-
-print(f"Batch data: {batch_data}")
-
-monitor_fg.insert(batch_data, write_options={"wait_for_job": True})
-monitoring_df = monitor_fg.filter(monitor_fg.days_before_forecast_day == 1).read()
-
-# Hindcast monitoring
-
-air_quality_fg = fs.get_feature_group(
-    name='air_quality',
-    version=1,
-)
-air_quality_df = air_quality_fg.read()
-
-outcome_df = air_quality_df[['date', 'pm25']]
-preds_df =  monitoring_df[['date', 'predicted_pm25']]
-hindcast_df = pd.merge(preds_df, outcome_df, on="date")
-hindcast_df = hindcast_df.sort_values(by=['date'])
-
-if len(hindcast_df) == 0:
-    hindcast_df = util.backfill_predictions_for_monitoring(weather_fg, air_quality_df, monitor_fg, retrieved_xgboost_model)
-
-plt = util.plot_air_quality_forecast(city, street, hindcast_df, file_path="./img/pm25_hindcast_1day.png", hindcast=True)

outcome_df.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:72e00d86aa97c8335b0ee61c6810ae6c5c0602dede91950641f5da0eec829b9a
-size 40620

requirements.txt

@@ -21,9 +21,6 @@ matplotlib==3.8.3
 
 hopsworks
 pandas
-#modal
 streamlit
 plotly
-
-apscheduler
 huggingface_hub

schedule.py

@@ -1,39 +0,0 @@
-import modal
-import requests
-import os
-import sys
-
-image = (
-    modal.Image.debian_slim(python_version="3.10")
-    .pip_install("requests","python-dotenv")
-)
-
-app = modal.App("build-scheduler",image=image)
-SPACE_ID = "Robzy/hbg-weather"  # Replace with your Space ID
-
-# Define a Modal function
-@app.function(schedule=modal.Period(hours=0, minutes=2),
-              secrets=[modal.Secret.from_dotenv()])  # Run every 2 minutes
-def trigger_rebuild():
-    import os
-    import requests
-
-    token = os.environ['HF_TOKEN']  # Your Hugging Face token
-    repo_id = "Robzy/hbg-weather"  # Replace with your Space's repo ID
-
-    headers = {
-        "Authorization": f"Bearer {token}",
-        "Content-Type": "application/json"
-    }
-
-    url = f"https://api.huggingface.co/spaces/{repo_id}/rebuild"
-
-    response = requests.post(url, headers=headers)
-
-    if response.status_code == 200:
-        print("Space rebuild triggered successfully!")
-    else:
-        print(f"Failed to trigger rebuild: {response.status_code}, {response.text}")
-
-if __name__ == "__main__":
-    trigger_rebuild()