Merge branch 'main' of https://github.com/regulatorystudies/cra-window-rules

app.py

@@ -1,6 +1,9 @@
 # ----- IMPORTS ----- #
 
 
+# ----- IMPORTS ----- #
+
+
 import asyncio
 from datetime import datetime, date, time
 from pathlib import Path
@@ -24,6 +27,7 @@ from modules import (
     plot_agency, 
     plot_tf, 
     plot_NA, 
+    plot_NA, 
     )
 
 from shiny import reactive
@@ -36,9 +40,18 @@ ui.include_css( Path(__file__).parent.joinpath("www") / "style.css")
 # ----- CREATE OBJECTS ----- #
 
 
+# this text appears in the browser tab
+# load css styles from external file
+ui.include_css( Path(__file__).parent.joinpath("www") / "style.css")
+
+
+# ----- CREATE OBJECTS ----- #
+
+
 # this text appears in the browser tab
 TITLE = "CRA Window Tracker - GW Regulatory Studies Center"
 
+# page header above main content
 # page header above main content
 HEADER = "Rules in the Congressional Review Act (CRA) Window"
 page_header = ui.HTML(
@@ -49,6 +62,15 @@ page_header = ui.HTML(
     """
     )
 
+# logo at the top of the sidebar
+page_header = ui.HTML(
+    f"""
+    <div class="header">
+        <span>{HEADER}</span>
+    </div>
+    """
+    )
+
 # logo at the top of the sidebar
 sidebar_logo = ui.HTML(
     f"""
@@ -71,10 +93,23 @@ FOOTER = f"""
 # ----- APP LAYOUT ----- #
 
 
+ui.tags.title(TITLE)
+# footer at the bottom of the page
+FOOTER = f"""
+    -----
+    
+    &copy; 2024 [GW Regulatory Studies Center](https://go.gwu.edu/regstudies). See our page on the [Congressional Review Act](https://regulatorystudies.columbian.gwu.edu/congressional-review-act) for more information.
+    """
+
+
+# ----- APP LAYOUT ----- #
+
+
 ui.tags.title(TITLE)
 
 page_header
 
+# sidebar settings
 # sidebar settings
 with ui.sidebar(open={"desktop": "open", "mobile": "closed"}):
     sidebar_logo
@@ -92,15 +127,19 @@ with ui.sidebar(open={"desktop": "open", "mobile": "closed"}):
 
     #ui.input_switch("switch", "Stack significant rules in plots", False)
 
+# value boxes with summary data
 # value boxes with summary data
 with ui.layout_column_wrap():
+    with ui.value_box():
     with ui.value_box():
         "All final rules"
         @render.text
         def count_rules():
             return f"{filtered_df()['document_number'].count()}"
         ui.input_action_button("filter_all", "Clear Filters", class_="btn-filter", icon=icon_svg("book"))
+        ui.input_action_button("filter_all", "Clear Filters", class_="btn-filter", icon=icon_svg("book"))
     
+    with ui.value_box():
     with ui.value_box():
         "Section 3(f)(1) Significant rules *"
         @render.text
@@ -110,7 +149,9 @@ with ui.layout_column_wrap():
                 output = f"{filtered_df()['3f1_significant'].sum()}"
             return output
         ui.input_action_button("filter_3f1", "Filter", class_="btn-filter", icon=icon_svg("book"))
+        ui.input_action_button("filter_3f1", "Filter", class_="btn-filter", icon=icon_svg("book"))
     
+    with ui.value_box():
     with ui.value_box():
         "Other Significant rules *"
         @render.text
@@ -120,7 +161,9 @@ with ui.layout_column_wrap():
                 output = f"{filtered_df()['other_significant'].sum()}"
             return output
         ui.input_action_button("filter_other", "Filter", class_="btn-filter", icon=icon_svg("book"))
+        ui.input_action_button("filter_other", "Filter", class_="btn-filter", icon=icon_svg("book"))
 
+# documentation note on significance data
 # documentation note on significance data
 ui.markdown(
             f"""
@@ -128,6 +171,7 @@ ui.markdown(
             """
             )
 
+# main content
 # main content
 with ui.navset_card_underline(title=""):
     
@@ -162,11 +206,14 @@ with ui.navset_card_underline(title=""):
                     max_val = max(values, default=0)
                     if (max_val < 2) or (count_gte_zero < 2):
                         return plot_NA()
+                        return plot_NA()
                     else:
                         return plot_tf(
                             grouped, 
                             input.frequency(), 
                             rule_type=filter_value.get(),
+                            input.frequency(), 
+                            rule_type=filter_value.get(),
                             )
             
             with ui.card(full_screen=True):
@@ -211,6 +258,14 @@ with ui.navset_card_underline(title=""):
                                 rule_type=filter_value.get(), 
                                 )
                         return plot
+                    if len(grouped) < 2:
+                        return plot_NA()
+                    else:
+                        plot = plot_agency(
+                                grouped.head(10), 
+                                rule_type=filter_value.get(), 
+                                )
+                        return plot
 
             with ui.card(full_screen=True):
                 @render.data_frame
@@ -225,6 +280,7 @@ with ui.navset_card_underline(title=""):
                         ]
                     return render.DataTable(grouped.loc[:, [c for c in cols if c in grouped.columns]])
 
+# download data
 # download data
 with ui.accordion(open=False):
     
@@ -259,6 +315,7 @@ with ui.accordion(open=False):
             await asyncio.sleep(0.25)
             yield filtered_df().loc[:, output_cols].to_csv(index=False)
 
+# notes
 # notes
 with ui.accordion(open=False):
     
@@ -272,6 +329,7 @@ with ui.accordion(open=False):
             """
             )
 
+# footer citation
 # footer citation
 ui.markdown(
     FOOTER

modules/plotting.py

@@ -4,6 +4,9 @@ from plotnine import (
     aes, 
     geom_col, 
     geom_line, 
+    annotate, 
+    theme, 
+    element_blank, 
     labs, 
     coord_flip, 
     scale_x_discrete, 
@@ -18,7 +21,30 @@ class DataAvailabilityError(Exception):
     pass
 
 
-def plot_agency(df, group_col = "acronym", value_col = "rules"):
+def plot_NA(placeholder_text: str = "Not enough data available to visualize.", placeholder_size: int = 14):
+    """Placeholder plot for when there is not enough data available to visualize."""
+    return (
+        ggplot() 
+        + annotate("text", x=0, y=0, label=placeholder_text, size=placeholder_size)
+        + theme(axis_ticks=element_blank(), axis_text=element_blank(), panel_grid=element_blank())
+        + labs(x="", y="", title="")
+        )
+
+
+def generate_rule_axis_label(rule_type: str | None = None):
+    """Generate axis label for rules, accounting for rule type ("all", "3f1", or "other")."""
+    y_lab = ""
+    if rule_type is not None:
+        rule_type_options = {
+            "all": "", 
+            "3f1": "3(f)(1) Significant",
+            "other": "Other Significant", 
+        }
+        y_lab = f"Number of {rule_type_options.get(rule_type)} rules".replace("  ", " ")
+    return y_lab
+
+
+def plot_agency(df, group_col = "acronym", value_col = "rules", color="#033C5A", rule_type: str | None = None):
     """Plot rules by agency.
 
     Args:
@@ -30,22 +56,32 @@ def plot_agency(df, group_col = "acronym", value_col = "rules"):
         ggplot: Plotted data.
     """
     order_list = df.loc[:, group_col].to_list()[::-1]
+
+    y_lab = generate_rule_axis_label(rule_type)
     
     plot = (
         ggplot(
             df, 
             aes(x=group_col, y=value_col), 
             )
-        + geom_col()
+        + geom_col(color="#FFFFFF", fill=color)
         + coord_flip()
         + scale_x_discrete(limits=order_list)
-        + labs(y="", x="", title="Number of Rules Published by Agency")
+        + labs(y=y_lab, x="", title="Rules Published by Agency")
         + theme_light()
         )
+
     return plot
 
 
-def plot_month(df: DataFrame, group_cols: tuple = ("publication_year", "publication_month"), value_col: str = "rules"):
+def plot_month(
+        df: DataFrame, 
+        group_cols: tuple = ("publication_year", "publication_month"), 
+        value_col: str = "rules", 
+        color: str = "#033C5A", 
+        title: str | None = None, 
+        y_lab: str = "",
+        ):
     """Plot rules by month.
 
     Args:
@@ -55,24 +91,33 @@ def plot_month(df: DataFrame, group_cols: tuple = ("publication_year", "publicat
 
     Returns:
         ggplot: Plotted data.
-    """    
+    """
     df.loc[:, "ym"] = df[group_cols[0]].astype(str) + "-" + df[group_cols[1]].astype(str).str.pad(2, fillchar="0")
     order_list = df.loc[:, "ym"].to_list()
-    
+    if title is None:
+        title = "Rules Published by Month"
+
     plot = (
         ggplot(
             df, 
             aes(x="ym", y=value_col), 
             )
-        + geom_col()
+        + geom_col(color="#FFFFFF", fill=color)
         + scale_x_discrete(limits=order_list)
-        + labs(y="", x="", title="Number of Rules Published by Month")
+        + labs(y=y_lab, x="", title=title)
         + theme_light()
         )
     return plot
 
 
-def plot_day(df: DataFrame, group_col: str = "publication_date", value_col: str = "rules"):
+def plot_day(
+        df: DataFrame, 
+        group_col: str = "publication_date", 
+        value_col: str = "rules", 
+        color: str = "#033C5A", 
+        title: str | None = None, 
+        y_lab: str = "",
+    ):
     """Plot rules by day.
 
     Args:
@@ -94,22 +139,33 @@ def plot_day(df: DataFrame, group_col: str = "publication_date", value_col: str
         freq = "1 month"
     
     max_value = df.loc[:, value_col].max()
+
+    if title is None:
+        title = "Rules Published by Date"
     
     plot = (
         ggplot(
             df, 
             aes(x=group_col, y=value_col), 
             )
-        + geom_line(group=1)
+        + geom_line(group=1, color=color)
         + scale_x_datetime(date_breaks=freq, date_labels="%m-%d")
         + scale_y_continuous(limits=(0, max_value), expand=(0, 0, 0.1, 0))
-        + labs(y="", x="", title="Number of Rules Published by Date")
+        + labs(y=y_lab, x="", title=title)
         + theme_light()
         )
     return plot
 
 
-def plot_week(df: DataFrame, group_col: str = "week_of", value_col: str = "rules", ):
+def plot_week(
+        df: DataFrame, 
+        group_col: str = "week_of", 
+        value_col: str = "rules", 
+        color: str = "#033C5A", 
+        title: str | None = None, 
+        y_lab: str = "",
+        show_significant: bool = False, 
+    ):
     max_value = df.loc[:, value_col].max()
     
     date_values = df[group_col].to_list()
@@ -128,21 +184,35 @@ def plot_week(df: DataFrame, group_col: str = "week_of", value_col: str = "rules
     
     breaks = [val for idx, val in enumerate(date_values) if idx % reduce_by == 0]
 
+    if title is None:
+        title = "Rules Published by Week"
+
     plot = (
         ggplot(
             df, 
             aes(x=group_col, y=value_col), 
             )
-        + geom_line(group=1)
+        + geom_line(group=1, color=color)
         + scale_x_datetime(breaks=breaks, labels=[f"{w.strftime('%m-%d')}" for w in breaks])
         + scale_y_continuous(limits=(0, max_value), expand=(0, 0, 0.1, 0))
-        + labs(y="", x="", title="Number of Rules Published by Week")
+        + labs(y=y_lab, x="", title=title)
         + theme_light()
         )
+    if show_significant:
+        # trying to add significant rules as additional lines
+        # but getting "TypeError: Discrete value supplied to continuous scale"
+        # for 3f1 sig rules
+        df = df.astype({"3f1_significant": "float"})
+        plot = (
+            plot
+            #+ geom_line(aes(x=group_col, y="3f1_significant"), inherit_aes=False, group=1, color="#AA9868", linetype="dotted")
+            + geom_line(aes(x=group_col, y="other_significant"), inherit_aes=False, group=1, color="#0190DB", linetype="dashed")
+            #+ guide_legend()
+        )
     return plot
 
 
-def plot_tf(df: DataFrame, frequency: str, **kwargs) -> ggplot:
+def plot_tf(df: DataFrame, frequency: str, rule_type: str | None = None, **kwargs) -> ggplot:
     """Plot rules over time by given frequency.
 
     Args:
@@ -160,8 +230,10 @@ def plot_tf(df: DataFrame, frequency: str, **kwargs) -> ggplot:
         "daily": plot_day, 
         "weekly": plot_week, 
         }
-    
     plot_freq = freq_options.get(frequency, None)
     if plot_freq is None:
         raise ValueError(f"Frequency must be one of: {', '.join(freq_options.keys())}")
-    return plot_freq(df, **kwargs)
+    
+    y_lab = generate_rule_axis_label(rule_type)
+
+    return plot_freq(df, y_lab=y_lab, **kwargs)

requirements-dev-mac.txt

@@ -121,7 +121,7 @@ idna==3.7
     #   requests
 iniconfig==2.0.0
     # via pytest
-ipykernel==6.29.4
+ipykernel==6.29.5
     # via
     #   -r requirements-dev.in
     #   jupyter
@@ -150,7 +150,7 @@ json5==0.9.25
     # via jupyterlab-server
 jsonpointer==3.0.0
     # via jsonschema
-jsonschema[format-nongpl]==4.22.0
+jsonschema[format-nongpl]==4.23.0
     # via
     #   jupyter-events
     #   jupyterlab-server
@@ -183,7 +183,7 @@ jupyter-events==0.10.0
     # via jupyter-server
 jupyter-lsp==2.2.5
     # via jupyterlab
-jupyter-server==2.14.1
+jupyter-server==2.14.2
     # via
     #   jupyter-lsp
     #   jupyterlab
@@ -196,7 +196,7 @@ jupyterlab==4.2.3
     # via notebook
 jupyterlab-pygments==0.3.0
     # via nbconvert
-jupyterlab-server==2.27.2
+jupyterlab-server==2.27.3
     # via
     #   jupyterlab
     #   notebook
@@ -417,7 +417,7 @@ rfc3986-validator==0.1.1
     # via
     #   jsonschema
     #   jupyter-events
-rpds-py==0.18.1
+rpds-py==0.19.0
     # via
     #   jsonschema
     #   referencing

requirements-dev-win.txt

@@ -78,7 +78,7 @@ cycler==0.12.1
     # via
     #   -c requirements.txt
     #   matplotlib
-debugpy==1.8.1
+debugpy==1.8.2
     # via ipykernel
 decorator==5.1.1
     # via ipython
@@ -125,7 +125,7 @@ idna==3.7
     #   requests
 iniconfig==2.0.0
     # via pytest
-ipykernel==6.29.0
+ipykernel==6.29.5
     # via
     #   -r requirements-dev.in
     #   jupyter
@@ -154,7 +154,7 @@ json5==0.9.25
     # via jupyterlab-server
 jsonpointer==3.0.0
     # via jsonschema
-jsonschema[format-nongpl]==4.22.0
+jsonschema[format-nongpl]==4.23.0
     # via
     #   jupyter-events
     #   jupyterlab-server
@@ -187,7 +187,7 @@ jupyter-events==0.10.0
     # via jupyter-server
 jupyter-lsp==2.2.5
     # via jupyterlab
-jupyter-server==2.14.1
+jupyter-server==2.14.2
     # via
     #   jupyter-lsp
     #   jupyterlab
@@ -196,11 +196,11 @@ jupyter-server==2.14.1
     #   notebook-shim
 jupyter-server-terminals==0.5.3
     # via jupyter-server
-jupyterlab==4.2.2
+jupyterlab==4.2.4
     # via notebook
 jupyterlab-pygments==0.3.0
     # via nbconvert
-jupyterlab-server==2.27.2
+jupyterlab-server==2.27.3
     # via
     #   jupyterlab
     #   notebook
@@ -339,7 +339,7 @@ prompt-toolkit==3.0.36
     #   ipython
     #   jupyter-console
     #   questionary
-psutil==5.9.8
+psutil==6.0.0
     # via ipykernel
 pure-eval==0.2.2
     # via stack-data
@@ -422,7 +422,7 @@ rfc3986-validator==0.1.1
     # via
     #   jsonschema
     #   jupyter-events
-rpds-py==0.18.1
+rpds-py==0.19.0
     # via
     #   jsonschema
     #   referencing

requirements.in

@@ -5,7 +5,7 @@ fr-toolbelt>=0.1.2, <1.0
 numpy>=1.26, <2.0
 pandas>=2.2, <3.0
 plotnine>=0.13.6, <1.0
-polars>=0.20.26, <1.0
+polars>=0.20.31
 pyarrow>=16.1.0, <17.0
 python-dateutil>=2.9.0.post0, <3.0
 requests>=2.32.2, <3.0

requirements.txt

@@ -4,7 +4,7 @@
 #
 #    pip-compile requirements.in
 #
-anyio==4.3.0
+anyio==4.4.0
     # via
     #   starlette
     #   watchfiles
@@ -30,13 +30,13 @@ cycler==0.12.1
     # via matplotlib
 faicons==0.2.2
     # via -r requirements.in
-fonttools==4.51.0
+fonttools==4.53.1
     # via matplotlib
 fr-toolbelt==0.1.3
     # via -r requirements.in
 h11==0.14.0
     # via uvicorn
-htmltools==0.5.1
+htmltools==0.5.3
     # via
     #   faicons
     #   shiny
@@ -52,13 +52,13 @@ markdown-it-py==3.0.0
     # via
     #   mdit-py-plugins
     #   shiny
-matplotlib==3.8.4
+matplotlib==3.9.1
     # via plotnine
 mdit-py-plugins==0.4.1
     # via shiny
 mdurl==0.1.2
     # via markdown-it-py
-mizani==0.11.3
+mizani==0.11.4
     # via plotnine
 numpy==1.26.4
     # via
@@ -73,7 +73,7 @@ numpy==1.26.4
     #   pyarrow
     #   scipy
     #   statsmodels
-packaging==24.0
+packaging==24.1
     # via
     #   htmltools
     #   matplotlib
@@ -88,11 +88,11 @@ pandas==2.2.2
     #   statsmodels
 patsy==0.5.6
     # via statsmodels
-pillow==10.3.0
+pillow==10.4.0
     # via matplotlib
 plotnine==0.13.6
     # via -r requirements.in
-polars==0.20.31
+polars==1.2.1
     # via -r requirements.in
 progress==1.6
     # via fr-toolbelt
@@ -117,7 +117,7 @@ requests==2.32.3
     # via
     #   -r requirements.in
     #   fr-toolbelt
-scipy==1.13.0
+scipy==1.14.0
     # via
     #   mizani
     #   plotnine
@@ -134,7 +134,7 @@ starlette==0.37.2
     # via shiny
 statsmodels==0.14.2
     # via plotnine
-typing-extensions==4.11.0
+typing-extensions==4.12.2
     # via
     #   htmltools
     #   shiny
@@ -146,9 +146,9 @@ uc-micro-py==1.0.3
     # via linkify-it-py
 urllib3==2.2.2
     # via requests
-uvicorn==0.29.0
+uvicorn==0.30.1
     # via shiny
-watchfiles==0.21.0
+watchfiles==0.22.0
     # via shiny
 wcwidth==0.2.13
     # via prompt-toolkit

www/style.css

@@ -15,3 +15,9 @@
   font-size: 30px;
   vertical-align: middle;
 }
+
+.btn-filter {
+  font-size: medium;
+  color: #FFFFFF;
+  background-color: #0190DB;
+}