app.py

# ----- IMPORTS ----- #


import asyncio
from datetime import datetime, date, time
from pathlib import Path

from pandas import DataFrame
from numpy import array

from modules import (
    DF, 
    LAST_UPDATED, 
    START_DATE, 
    WINDOW_OPEN_DATE, 
    GET_SIGNIFICANT, 
    METADATA, 
    AGENCIES,
    CRA_LAST_UPDATED,  
    groupby_agency, 
    groupby_date, 
    add_week_info_to_data, 
    pad_missing_dates, 
    plot_agency, 
    plot_tf, 
    plot_NA, 
    plot_NA,
    get_rd
    )

from shiny import reactive
from shiny.express import input, render, ui

# load css styles from external file
ui.include_css( Path(__file__).parent.joinpath("www") / "style.css")


# ----- CREATE OBJECTS ----- #


# this text appears in the browser tab
TITLE = "CRA Window Exploratory Dashboard - GW Regulatory Studies Center"

# page header above main content
HEADER = "Congressional Review Act (CRA) Window Exploratory Dashboard"
page_header = ui.HTML(
    f"""
    <div class="header">
        <h1>{HEADER}</h1>
    </div>
    """
    )

# logo at the top of the sidebar
sidebar_logo = ui.HTML(
    f"""
    <div class="header">
        <a href="https://go.gwu.edu/regstudies" target="_blank">
        <img src="logo.png" alt="Regulatory Studies Center logo"/>        
        </a>
    </div>
    """
    )

# footer at the bottom of the page
FOOTER = f"""
    -----
    
    &copy; {date.today().year} [GW Regulatory Studies Center](https://go.gwu.edu/regstudies). See our page on the [Congressional Review Act](https://regulatorystudies.columbian.gwu.edu/congressional-review-act) for more information.
    """


# ----- APP LAYOUT ----- #


ui.tags.title(TITLE)

page_header

# sidebar settings
with ui.sidebar(open={"desktop": "open", "mobile": "closed"}, fg="#033C5A"):
    sidebar_logo

    with ui.tooltip(placement="right", id="window_tooltip"):
        ui.input_date("start_date", "Select start of window", value=WINDOW_OPEN_DATE, 
                      min=START_DATE, max=datetime(date.fromisoformat(WINDOW_OPEN_DATE).year+1,1,3))
        "The CRA lookback window opens on August 16, 2024. Select a different date to explore how different lookback dates would affect the set of rules available for congressional review. See the notes for more information."

    with ui.tooltip(placement="right", id="window_end_tooltip"):
        ui.input_date("end_date", "Select end of window", value=datetime(date.fromisoformat(WINDOW_OPEN_DATE).year+1,1,3), 
                      min=datetime(date.fromisoformat(WINDOW_OPEN_DATE).year+1,1,3), max=datetime(date.fromisoformat(WINDOW_OPEN_DATE).year+1,1,20))
        "The default end date of the CRA lookback window is set to January 3, when the next session of Congress begins. You may select a different date, up to January 20, to explore rules published between the end of the previous session of Congress and the inauguration date (if applicable)."

    with ui.tooltip(placement="right", id="sig_tooltip"):
        ui.input_select("menu_significant", "Select rule significance", choices=["all", "3f1-significant", "other-significant"], selected="all", multiple=True, size=3)
        "Rule significance as defined in Executive Order 12866, as amended by Executive Order 14094."

    with ui.tooltip(placement="right", id="cra_tooltip"):
        ui.input_select("menu_cra_target", "Select CRA target status", choices=["all", "CRA targeted", "Not CRA targeted"], selected="all", multiple=True, size=3)
        f"Whether the 119th Congress has introduced a joint resolution of disapproval (RD) targeting a rule (data last updated {CRA_LAST_UPDATED})."

    with ui.tooltip(placement="right", id="agency_tooltip"):
        ui.input_select("menu_agency", "Select agencies", choices=["all"] + AGENCIES, selected=["all"], multiple=True, size=6)
        "Select one or more parent-level agencies."

# value boxes with summary data
with ui.layout_column_wrap():
    with ui.value_box(class_="summary-values"):
        "All final rules"
        with ui.tooltip(placement="bottom", id="all_tooltip"):
            @render.text
            def count_rules():
                return f"{filtered_df()['document_number'].count()}"
            f"Federal Register data last retrieved {date.today()}."
    
    with ui.value_box(class_="summary-values"):
        "Section 3(f)(1) Significant rules"
        with ui.tooltip(placement="bottom", id="3f1_tooltip"):
            @render.text
            def count_3f1_significant():
                output = "Not available"
                if GET_SIGNIFICANT:
                    output = f"{filtered_df()['3f1_significant'].sum()}"
                return output
            f"Executive Order 12866 significance data last updated {LAST_UPDATED}."

    with ui.value_box(class_="summary-values"):
        "Other Significant rules"
        with ui.tooltip(placement="bottom", id="other_tooltip"):
            @render.text
            def count_other_significant():
                output = "Not available"
                if GET_SIGNIFICANT:
                    output = f"{filtered_df()['other_significant'].sum()}"
                return output
            f"Executive Order 12866 significance data last updated {LAST_UPDATED}."

# main content
with ui.navset_card_underline(title=""):
    
    with ui.nav_panel("Rules in detail"):
        with ui.card(full_screen=True):
            @render.data_frame
            def table_rule_detail():
                df = filter_significance().copy()
                df.loc[:, "date"] = df.loc[:, "publication_date"].apply(lambda x: f"{x.date()}")
                char, limit = " ", 10
                df.loc[:, "title"] = df["title"].apply(lambda x: x if len(x.split(char)) < (limit + 1) else f"{char.join(x.split(char)[:limit])}...")
                df.loc[:, "agencies"] = df["parent_slug"].apply(lambda x: "; ".join(x))
                cols = [
                    "date", 
                    "title", 
                    "agencies", 
                    "3f1_significant", 
                    "other_significant",
                    "CRA_Target",
                    "Latest_CRA_Stage",
                    "RD_No",
                    ]
                return render.DataGrid(df.loc[:, [c for c in cols if c in df.columns]], width="100%")

    with ui.nav_panel("By agency"):
        
        with ui.layout_columns():
            
            with ui.card(full_screen=True):
                @render.plot
                def plot_by_agency():
                    grouped = grouped_df_agency()
                    if len(grouped) < 2:
                        return plot_NA()
                    else:
                        plot = plot_agency(
                                grouped.head(10), 
                                rule_types=input.menu_significant(), 
                                )
                        return plot

            with ui.card(full_screen=True):
                @render.data_frame
                def table_by_agency():
                    grouped = grouped_df_agency()
                    cols = [
                        "agency", 
                        "acronym",
                        "rules",
                        "3f1_significant", 
                        "other_significant",
                        "cra_targeted",
                        ]
                    return render.DataTable(grouped.loc[:, [c for c in cols if c in grouped.columns]])

    with ui.nav_panel("Over time"):
        
        ui.input_select("frequency", "Select frequency", choices=["daily", "weekly", "monthly"], selected="monthly")
        
        with ui.layout_columns():
        
            with ui.card(full_screen=True):
                
                @render.plot
                def plot_over_time(value_col: str = "rules"):
                    grouped = get_grouped_data_over_time()
                    values = grouped.loc[:, value_col].to_numpy()
                    count_gte_zero = sum(1 if g > 0 else 0 for g in values)
                    max_val = max(values, default=0)
                    if (max_val < 2) or (count_gte_zero < 2):
                        return plot_NA()
                    else:
                        return plot_tf(
                            grouped, 
                            input.frequency(), 
                            rule_types=input.menu_significant(),
                            )
            
            with ui.card(full_screen=True):
                @render.data_frame
                def table_over_time():
                    grouped = get_grouped_data_over_time()
                    date_cols = ["publication_date", "week_of", ]
                    if any(d in grouped.columns for d in date_cols):
                        grouped = grouped.astype({d: "str" for d in date_cols if d in grouped.columns}, errors="ignore")
                    grouped = grouped.rename(columns={
                        "publication_year": "year", 
                        "publication_month": "month", 
                        "publication_date": "date", 
                        }, errors="ignore")
                    cols = [
                        "date", 
                        "year", 
                        "month", 
                        "week_of", 
                        "rules",
                        "3f1_significant", 
                        "other_significant",
                        "cra_targeted",
                        ]
                    return render.DataTable(grouped.loc[:, [c for c in cols if c in grouped.columns]])

# download data
with ui.accordion(open=False):
    
    with ui.accordion_panel("Download Data"):
        
        @render.download(
            label="Download data as CSV",
            filename=f"rules_in_cra_window_accessed_{date.today()}.csv",
        )
        async def download(
            output_cols: tuple | list = (
                "document_number", 
                "citation", 
                "publication_date", 
                "title", 
                "type", 
                "action", 
                "json_url", 
                "html_url", 
                "agencies", 
                "independent_reg_agency", 
                "parent_agencies", 
                "subagencies", 
                "president_id", 
                "significant", 
                "3f1_significant", 
                "other_significant",
                "CRA_Target",
                "Latest_CRA_Stage",
                )
            ):
            filt_df = filter_significance().copy()
            filt_df.loc[:, "agencies"] = filt_df.loc[:, "agency_slugs"].apply(lambda x: "; ".join(x))
            filt_df.loc[:, "parent_agencies"] = filt_df.loc[:, "parent_slug"].apply(lambda x: "; ".join(x))
            filt_df.loc[:, "subagencies"] = filt_df.loc[:, "subagency_slug"].apply(lambda x: "; ".join(x))
            rd_cols=[col for col in filt_df.columns if col.startswith('RD') and '_' in col and col.split('_')[0][2:].isdigit()]
            await asyncio.sleep(0.25)
            yield filt_df.loc[:, [c for c in list(output_cols)+rd_cols if c in filt_df.columns]].to_csv(index=False)

# notes
with ui.accordion(open=False):
    
    with ui.accordion_panel("Notes"):
        
        ui.markdown(
            f"""            
            This dashboard allows users to explore how different [Congressional Review Act](https://uscode.house.gov/view.xhtml?req=granuleid%3AUSC-prelim-title5-chapter8&saved=%7CKHRpdGxlOjUgc2VjdGlvbjo4MDEgZWRpdGlvbjpwcmVsaW0pIE9SIChncmFudWxlaWQ6VVNDLXByZWxpbS10aXRsZTUtc2VjdGlvbjgwMSk%3D%7CdHJlZXNvcnQ%3D%7C%7C0%7Cfalse%7Cprelim&edition=prelim) (CRA) lookback window dates would affect the set of rules available for congressional review and tracks resolutions for disapproval (RDs) targeting the rules published within this window. 
            The “lookback window” refers to the period starting [60 working days](https://crsreports.congress.gov/product/pdf/R/R46690#page=8) (either session days in the Senate or legislative days in the House of Representatives) before the current session of Congress adjourns and ending the day the subsequent session of Congress first convenes. 
            Rules that are published in the Federal Register and submitted to Congress after the lookback day are made available for review in the subsequent session of Congress. 
            The current lookback date is [August 16, 2024](https://www.congress.gov/congressional-record/volume-171/issue-18/house-section/article/H398-8). 
            
            "Section 3(f)(1) significant" rules are regulations that meet the criteria in Section 3(f)(1) of [Executive Order 12866](https://www.archives.gov/files/federal-register/executive-orders/pdf/12866.pdf), as amended by [Executive Order 14094](https://www.govinfo.gov/content/pkg/FR-2023-04-11/pdf/2023-07760.pdf), referring to those with an estimated annual effect on the economy of $200 million or more.
            "Other significant" rules are regulations that meet the other criteria in Section 3(f) of Executive Order 12866, as amended by Executive Order 14094, such as those creating inconsistency with other agencies' actions, altering certain budgetary impacts, or raising legal or policy issues pertaining to the president's priorities.

            Rule data are retrieved daily from the [Federal Register API](https://www.federalregister.gov/developers/documentation/api/v1), which publishes new editions of the Federal Register each business day. 
            """
            )

# footer citation
ui.markdown(
    FOOTER
)


# ----- REACTIVE CALCULATIONS ----- #


@reactive.calc
def filtered_df(agency_column: str = "parent_slug"):
    filt_df = DF

    # merge with RD data
    filt_df = get_rd(filt_df)

    # filter dates
    try:
        filt_df = filt_df.loc[(filt_df["publication_date"] >= input.start_date()) & (filt_df["publication_date"] <= input.end_date())]
    except TypeError:
        filt_df = filt_df.loc[(filt_df["publication_date"] >= datetime.combine(input.start_date(), time(0, 0))) & 
                              (filt_df["publication_date"] <= datetime.combine(input.end_date(), time(0, 0)))]
    
    # filter agencies
    if (input.menu_agency() is not None) and ("all" not in input.menu_agency()):
        bool_agency = [True if sum(selected in agency for selected in input.menu_agency()) > 0 else False for agency in filt_df[agency_column]]
        filt_df = filt_df.loc[bool_agency]

    # filter CRA target status
    bool_cra = []
    if (input.menu_cra_target() is not None) and ("all" not in input.menu_cra_target()):
        if "CRA targeted" in input.menu_cra_target():
            bool_cra.append((filt_df["CRA_Target"] == 1).to_numpy())
        if "Not CRA targeted" in input.menu_cra_target():
            bool_cra.append((filt_df["CRA_Target"] == 0).to_numpy())
        filt_df = filt_df.loc[array(bool_cra).any(axis=0)]

    # return filtered dataframe
    return filt_df


@reactive.calc
def filter_significance():

    # get data filtered by date and agency
    filt_df = filtered_df()

    # filter significance
    bool_ = []
    if (input.menu_significant() is not None) and ("all" not in input.menu_significant()):
        if "3f1-significant" in input.menu_significant():
            bool_.append((filt_df["3f1_significant"] == 1).to_numpy())
        if "other-significant" in input.menu_significant():
            bool_.append((filt_df["other_significant"] == 1).to_numpy())
        filt_df = filt_df.loc[array(bool_).any(axis=0)]

    # return filtered dataframe
    return filt_df


@reactive.calc
def grouped_df_month():
    filt_df = filter_significance()
    grouped = groupby_date(filt_df, significant=GET_SIGNIFICANT)
    return grouped


@reactive.calc
def grouped_df_day():
    filt_df = filter_significance()
    date_col = "publication_date"
    grouped = groupby_date(filt_df, group_col=date_col, significant=GET_SIGNIFICANT)
    grouped = pad_missing_dates(
        grouped, 
        date_col, 
        "days", 
        fill_padded_values={
            "rules": 0, 
            "3f1_significant": 0, 
            "other_significant": 0,
            "cra_targeted":0, 
            })
    return grouped


@reactive.calc
def grouped_df_week():
    filt_df = filter_significance()
    filt_df = add_week_info_to_data(filt_df)
    try:
        grouped = groupby_date(filt_df, group_col=("week_number", "week_of"), significant=GET_SIGNIFICANT)
        grouped = pad_missing_dates(
            grouped, 
            "week_of", 
            how="weeks", 
            fill_padded_values={
                "rules": 0, 
                "3f1_significant": 0, 
                "other_significant": 0,
                "cra_targeted":0 
                })
    except KeyError as err:
        grouped = DataFrame(columns=["week_number", "week_of", "rules", "3f1_significant", "other_significant","cra_targeted"])
    return grouped


@reactive.calc
def grouped_df_agency():
    filt_df=filter_significance()
    grouped = groupby_agency(filt_df, metadata=METADATA, significant=GET_SIGNIFICANT)
    return grouped


@reactive.calc
def get_grouped_data_over_time():
    if input.frequency() == "daily":
        grouped = grouped_df_day()
    elif input.frequency() == "monthly":
        grouped = grouped_df_month()
    elif input.frequency() == "weekly":
        grouped = grouped_df_week()
    else:
        raise ValueError("Only 'daily', 'monthly', or 'weekly' are valid inputs.")
    return grouped