updating README

.github/update-df.yml

@@ -1,34 +1,34 @@
-# name: update-df
+name: update-df
 
-# on:
-#   workflow_dispatch:
-#     schedule:
-#     - cron: '11 6 * * *'
+on:
+  workflow_dispatch:
+    schedule:
+    - cron: '11 6 * * *'
 
-# jobs:
-#   schedule_pipelines:
-#     runs-on: ubuntu-latest
+jobs:
+  schedule_pipelines:
+    runs-on: ubuntu-latest
 
-#     permissions:
-#       pages: write
-#       contents: write
+    permissions:
+      pages: write
+      contents: write
 
-#     steps:
-#       - name: checkout repo content
-#         uses: actions/checkout@v4
+    steps:
+      - name: checkout repo content
+        uses: actions/checkout@v4
 
-#       - name: setup python
-#         uses: actions/setup-python@v5
-#         with:
-#           python-version: '3.10'
-#           cache: 'pip'
-#       - name: install python packages
-#         run: |
-#           python -m pip install --upgrade pip
-#           pip install -r requirements.txt
+      - name: setup python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+          cache: 'pip'
+      - name: install python packages
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
 
-#       - name: execute python workflows from bash script
-#         env:
-#           HOPSWORKS_API_KEY: ${{ secrets.HOPSWORKS_API_KEY }}
-#         run: |
-#           python infer.py
+      - name: execute python workflows from bash script
+        env:
+          HOPSWORKS_API_KEY: ${{ secrets.HOPSWORKS_API_KEY }}
+        run: |
+          python infer.py

README.md

@@ -1,5 +1,5 @@
 ---
-title: Hbg Weather
+title: Air Quality Forecast
 emoji: 😻
 colorFrom: blue
 colorTo: purple
@@ -7,9 +7,50 @@ sdk: streamlit
 sdk_version: 1.25.0
 app_file: app_streamlit.py
 pinned: false
-short_description: Weather prediction for Helsingborg, Sweden!
+short_description: Air quality forecasting for Lahore, Pakistan!
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Air Quality Monitoring for Lahore, Pakistan
 
-gradio version: 5.6.0
+### Dashboard link: https://huggingface.co/spaces/Robzy/hbg-weather
+
+# Architecture & pipeline
+
+### 1. Data sourcing
+
+* Historical air quality measurements are collected from World Air Quality Index in .csv form.
+* Historical weather data collected from Open-Meteo API client, loaded as Pandas dataframe. 
+
+Weather data features:
+
+ * Temperature (average over the day)
+ * Pecipitation (the total over the day)
+ * Wind speed (average over the day)
+ * Wind direction (the most dominant direction over the day)
+
+
+### 2. Backfill
+
+* Created two feature groups are created on Hopsworks: `air_quality` and  `weather`
+* Data us pre-processed and uploaded to their respective feature groups.
+
+### 3. Feature pipeline
+
+* Daily weather and air quality data is fetched from Open Meteo and World Air Quality Index APIs respectively, then uploaded into the feature groups. 
+
+### 4. Training & model
+
+* A feature view `air_quality_fv` is created on Hopsworks, which is an input/output API schema for a model.
+* We train a XGBoost regression model `air_quality_xgboost_model` and save it to our model registry.
+
+### 5. Inference pipeline
+
+* A new feature group `aq_predictions` is created on Hopsworks.
+* Upon an inference request, the forecasted features and the model are retrieved from the feature view and model regsitry respectively.
+* Air quality predictions are made by inputting the forecasted features into the model, and then uploaded into the `aq_predictions` feature group.
+
+# Dashboard & scheduling
+
+HuggingFace's Streamlit Spaces is used to display the hindcast, forecast, and real air quality using an interactive line graph. GitHub Actions is used to call the feature and inference pipeline daily by levraging schedulin.
+
+Note that backfilling, feature group creation and model training is only performed once

app_streamlit.py

@@ -19,8 +19,9 @@ st.set_page_config(
     }
 )
 
-st.title('Lahore Air Quality!')
-st.subheader('Particle matter, diameter < 2.5 micrometers (PM2.5)')
+st.title('Lahore Air Quality')
+st.subheader('Forecast and hindcast')
+st.subheader('Unit: PM25 - particle matter of diameter < 2.5 micrometers')
 
 #pickle_file_path = 'air_quality_df.pkl'
 pickle_file_path = 'outcome_df.pkl'

debug.ipynb

@@ -51,7 +51,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "metadata": {},
    "outputs": [
     {
@@ -90,136 +90,12 @@
     ")\n",
     "\n",
     "today_timestamp = pd.to_datetime(today)\n",
-    "batch_data = weather_fg.filter(weather_fg.date >= today_timestamp ).read()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 17,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "<div>\n",
-       "<style scoped>\n",
-       "    .dataframe tbody tr th:only-of-type {\n",
-       "        vertical-align: middle;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe tbody tr th {\n",
-       "        vertical-align: top;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe thead th {\n",
-       "        text-align: right;\n",
-       "    }\n",
-       "</style>\n",
-       "<table border=\"1\" class=\"dataframe\">\n",
-       "  <thead>\n",
-       "    <tr style=\"text-align: right;\">\n",
-       "      <th></th>\n",
-       "      <th>date</th>\n",
-       "      <th>temperature_2m_mean</th>\n",
-       "      <th>precipitation_sum</th>\n",
-       "      <th>wind_speed_10m_max</th>\n",
-       "      <th>wind_direction_10m_dominant</th>\n",
-       "      <th>city</th>\n",
-       "      <th>predicted_pm25</th>\n",
-       "      <th>street</th>\n",
-       "      <th>country</th>\n",
-       "      <th>days_before_forecast_day</th>\n",
-       "    </tr>\n",
-       "  </thead>\n",
-       "  <tbody>\n",
-       "    <tr>\n",
-       "      <th>0</th>\n",
-       "      <td>2024-11-21 00:00:00+00:00</td>\n",
-       "      <td>3.40</td>\n",
-       "      <td>0.2</td>\n",
-       "      <td>19.995398</td>\n",
-       "      <td>246.665939</td>\n",
-       "      <td>Helsingborg</td>\n",
-       "      <td>39.168438</td>\n",
-       "      <td>Drottninggatan</td>\n",
-       "      <td>Sweden</td>\n",
-       "      <td>1</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>3</th>\n",
-       "      <td>2024-11-22 00:00:00+00:00</td>\n",
-       "      <td>4.05</td>\n",
-       "      <td>0.7</td>\n",
-       "      <td>23.540806</td>\n",
-       "      <td>246.571289</td>\n",
-       "      <td>Helsingborg</td>\n",
-       "      <td>20.740093</td>\n",
-       "      <td>Drottninggatan</td>\n",
-       "      <td>Sweden</td>\n",
-       "      <td>4</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>2</th>\n",
-       "      <td>2024-11-23 00:00:00+00:00</td>\n",
-       "      <td>5.45</td>\n",
-       "      <td>0.0</td>\n",
-       "      <td>30.631746</td>\n",
-       "      <td>240.422256</td>\n",
-       "      <td>Helsingborg</td>\n",
-       "      <td>46.448105</td>\n",
-       "      <td>Drottninggatan</td>\n",
-       "      <td>Sweden</td>\n",
-       "      <td>3</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>1</th>\n",
-       "      <td>2024-11-24 00:00:00+00:00</td>\n",
-       "      <td>5.60</td>\n",
-       "      <td>0.0</td>\n",
-       "      <td>13.755580</td>\n",
-       "      <td>276.008911</td>\n",
-       "      <td>Helsingborg</td>\n",
-       "      <td>61.713448</td>\n",
-       "      <td>Drottninggatan</td>\n",
-       "      <td>Sweden</td>\n",
-       "      <td>2</td>\n",
-       "    </tr>\n",
-       "  </tbody>\n",
-       "</table>\n",
-       "</div>"
-      ],
-      "text/plain": [
-       "                       date  temperature_2m_mean  precipitation_sum  \\\n",
-       "0 2024-11-21 00:00:00+00:00                 3.40                0.2   \n",
-       "3 2024-11-22 00:00:00+00:00                 4.05                0.7   \n",
-       "2 2024-11-23 00:00:00+00:00                 5.45                0.0   \n",
-       "1 2024-11-24 00:00:00+00:00                 5.60                0.0   \n",
-       "\n",
-       "   wind_speed_10m_max  wind_direction_10m_dominant         city  \\\n",
-       "0           19.995398                   246.665939  Helsingborg   \n",
-       "3           23.540806                   246.571289  Helsingborg   \n",
-       "2           30.631746                   240.422256  Helsingborg   \n",
-       "1           13.755580                   276.008911  Helsingborg   \n",
-       "\n",
-       "   predicted_pm25          street country  days_before_forecast_day  \n",
-       "0       39.168438  Drottninggatan  Sweden                         1  \n",
-       "3       20.740093  Drottninggatan  Sweden                         4  \n",
-       "2       46.448105  Drottninggatan  Sweden                         3  \n",
-       "1       61.713448  Drottninggatan  Sweden                         2  "
-      ]
-     },
-     "execution_count": 17,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "batch_data.sort_values(by=['date'])"
+    "batch_data = weather_fg.filter(weather_fg.date >= today_timestamp ).read().sort_values(by=['date'])"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 18,
    "metadata": {},
    "outputs": [
     {
@@ -313,189 +189,6 @@
     "plt = util.plot_air_quality_forecast(city, street, hindcast_df, file_path=\"./img/pm25_hindcast_1day.png\", hindcast=True)"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": 13,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "<div>\n",
-       "<style scoped>\n",
-       "    .dataframe tbody tr th:only-of-type {\n",
-       "        vertical-align: middle;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe tbody tr th {\n",
-       "        vertical-align: top;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe thead th {\n",
-       "        text-align: right;\n",
-       "    }\n",
-       "</style>\n",
-       "<table border=\"1\" class=\"dataframe\">\n",
-       "  <thead>\n",
-       "    <tr style=\"text-align: right;\">\n",
-       "      <th></th>\n",
-       "      <th>date</th>\n",
-       "      <th>pm25</th>\n",
-       "    </tr>\n",
-       "  </thead>\n",
-       "  <tbody>\n",
-       "    <tr>\n",
-       "      <th>478</th>\n",
-       "      <td>2020-06-01</td>\n",
-       "      <td>12.0</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>746</th>\n",
-       "      <td>2020-06-02</td>\n",
-       "      <td>14.0</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>1211</th>\n",
-       "      <td>2020-06-03</td>\n",
-       "      <td>20.0</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>1536</th>\n",
-       "      <td>2020-06-04</td>\n",
-       "      <td>21.0</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>1470</th>\n",
-       "      <td>2020-06-05</td>\n",
-       "      <td>18.0</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>...</th>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>764</th>\n",
-       "      <td>2024-11-12</td>\n",
-       "      <td>26.0</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>1265</th>\n",
-       "      <td>2024-11-13</td>\n",
-       "      <td>35.0</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>1315</th>\n",
-       "      <td>2024-11-14</td>\n",
-       "      <td>22.0</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>301</th>\n",
-       "      <td>2024-11-15</td>\n",
-       "      <td>7.0</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>1588</th>\n",
-       "      <td>2024-11-16</td>\n",
-       "      <td>34.0</td>\n",
-       "    </tr>\n",
-       "  </tbody>\n",
-       "</table>\n",
-       "<p>1589 rows × 2 columns</p>\n",
-       "</div>"
-      ],
-      "text/plain": [
-       "            date  pm25\n",
-       "478   2020-06-01  12.0\n",
-       "746   2020-06-02  14.0\n",
-       "1211  2020-06-03  20.0\n",
-       "1536  2020-06-04  21.0\n",
-       "1470  2020-06-05  18.0\n",
-       "...          ...   ...\n",
-       "764   2024-11-12  26.0\n",
-       "1265  2024-11-13  35.0\n",
-       "1315  2024-11-14  22.0\n",
-       "301   2024-11-15   7.0\n",
-       "1588  2024-11-16  34.0\n",
-       "\n",
-       "[1589 rows x 2 columns]"
-      ]
-     },
-     "execution_count": 13,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "outcome_df.sort_values(by=['date'])"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 14,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "<div>\n",
-       "<style scoped>\n",
-       "    .dataframe tbody tr th:only-of-type {\n",
-       "        vertical-align: middle;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe tbody tr th {\n",
-       "        vertical-align: top;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe thead th {\n",
-       "        text-align: right;\n",
-       "    }\n",
-       "</style>\n",
-       "<table border=\"1\" class=\"dataframe\">\n",
-       "  <thead>\n",
-       "    <tr style=\"text-align: right;\">\n",
-       "      <th></th>\n",
-       "      <th>date</th>\n",
-       "      <th>predicted_pm25</th>\n",
-       "    </tr>\n",
-       "  </thead>\n",
-       "  <tbody>\n",
-       "    <tr>\n",
-       "      <th>1</th>\n",
-       "      <td>2024-11-15 00:00:00+00:00</td>\n",
-       "      <td>33.413746</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>0</th>\n",
-       "      <td>2024-11-16 00:00:00+00:00</td>\n",
-       "      <td>34.305458</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>2</th>\n",
-       "      <td>2024-11-21 00:00:00+00:00</td>\n",
-       "      <td>39.168438</td>\n",
-       "    </tr>\n",
-       "  </tbody>\n",
-       "</table>\n",
-       "</div>"
-      ],
-      "text/plain": [
-       "                       date  predicted_pm25\n",
-       "1 2024-11-15 00:00:00+00:00       33.413746\n",
-       "0 2024-11-16 00:00:00+00:00       34.305458\n",
-       "2 2024-11-21 00:00:00+00:00       39.168438"
-      ]
-     },
-     "execution_count": 14,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "preds_df.sort_values(by=['date'])"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": 23,

functions/figure.py

@@ -84,10 +84,13 @@ def plot(df, n=10):
         line=dict(width=3)     # Make the line thicker
     )
 
+    # x range start BEFORE today
+    k = 4
+
     fig.update_layout(
         shapes=shapes,  # Add the background rectangles
         xaxis=dict(
-            range=[x_values.iloc[-n], x_values.iloc[-1]],  # Dynamically set the range
+            range=[x_values.iloc[-n-k], x_values.iloc[-1]],  # Dynamically set the range
             title=dict(
                 text="Date",  # Set x-axis label
                 font=dict(size=label_font_size)  # Increase font size for the x-axis label