Add CRA filter

app.py

@@ -15,7 +15,8 @@ from modules import (
     WINDOW_OPEN_DATE, 
     GET_SIGNIFICANT, 
     METADATA, 
-    AGENCIES,  
+    AGENCIES,
+    CRA_LAST_UPDATED,  
     groupby_agency, 
     groupby_date, 
     add_week_info_to_data, 
@@ -24,7 +25,7 @@ from modules import (
     plot_tf, 
     plot_NA, 
     plot_NA,
-    merge_rd
+    get_rd
     )
 
 from shiny import reactive
@@ -65,7 +66,7 @@ sidebar_logo = ui.HTML(
 FOOTER = f"""
     -----
     
-    © 2024 [GW Regulatory Studies Center](https://go.gwu.edu/regstudies). See our page on the [Congressional Review Act](https://regulatorystudies.columbian.gwu.edu/congressional-review-act) for more information.
+    © {date.today().year} [GW Regulatory Studies Center](https://go.gwu.edu/regstudies). See our page on the [Congressional Review Act](https://regulatorystudies.columbian.gwu.edu/congressional-review-act) for more information.
     """
 
 
@@ -94,6 +95,10 @@ with ui.sidebar(open={"desktop": "open", "mobile": "closed"}, fg="#033C5A"):
         ui.input_select("menu_significant", "Select rule significance", choices=["all", "3f1-significant", "other-significant"], selected="all", multiple=True, size=3)
         "Rule significance as defined in Executive Order 12866, as amended by Executive Order 14094."
 
+    with ui.tooltip(placement="right", id="cra_tooltip"):
+        ui.input_select("menu_cra_target", "Select CRA target status", choices=["all", "CRA targeted", "Not CRA targeted"], selected="all", multiple=True, size=3)
+        f"Whether a rule has been targeted in a joint resolution (RD) for congressional disapproval (data last updated {CRA_LAST_UPDATED})."
+
     with ui.tooltip(placement="right", id="agency_tooltip"):
         ui.input_select("menu_agency", "Select agencies", choices=["all"] + AGENCIES, selected=["all"], multiple=True, size=6)
         "Select one or more parent-level agencies."
@@ -150,7 +155,7 @@ with ui.navset_card_underline(title=""):
                     "other_significant",
                     "CRA Target",
                     "CRA Stage",
-                    "RD Ref & Date", 
+                    "RD No. & Date", 
                     ]
                 return render.DataGrid(df.loc[:, [c for c in cols if c in df.columns]], width="100%")
     
@@ -199,7 +204,7 @@ with ui.navset_card_underline(title=""):
                         "other_significant",
                         "CRA Target",
                         "CRA Stage",
-                        "RD Ref & Date",
+                        "RD No. & Date",
                         ]
                     return render.DataTable(grouped.loc[:, [c for c in cols if c in grouped.columns]])
 
@@ -232,7 +237,7 @@ with ui.navset_card_underline(title=""):
                         "other_significant",
                         "CRA Target",
                         "CRA Stage",
-                        "RD Ref & Date",
+                        "RD No. & Date",
                         ]
                     return render.DataTable(grouped.loc[:, [c for c in cols if c in grouped.columns]])
 
@@ -265,11 +270,11 @@ with ui.accordion(open=False):
                 "other_significant",
                 "CRA Target",
                 "CRA Stage",
-                "RD Ref & Date",
+                "RD No. & Date",
                 "RD Link"
                 )
             ):
-            filt_df = filtered_df().copy()
+            filt_df = filter_significance().copy()
             filt_df.loc[:, "agencies"] = filt_df.loc[:, "agency_slugs"].apply(lambda x: "; ".join(x))
             filt_df.loc[:, "parent_agencies"] = filt_df.loc[:, "parent_slug"].apply(lambda x: "; ".join(x))
             filt_df.loc[:, "subagencies"] = filt_df.loc[:, "subagency_slug"].apply(lambda x: "; ".join(x))
@@ -283,10 +288,10 @@ with ui.accordion(open=False):
         
         ui.markdown(
             f"""            
-            The [Congressional Review Act](https://uscode.house.gov/view.xhtml?req=granuleid%3AUSC-prelim-title5-chapter8&saved=%7CKHRpdGxlOjUgc2VjdGlvbjo4MDEgZWRpdGlvbjpwcmVsaW0pIE9SIChncmFudWxlaWQ6VVNDLXByZWxpbS10aXRsZTUtc2VjdGlvbjgwMSk%3D%7CdHJlZXNvcnQ%3D%7C%7C0%7Cfalse%7Cprelim&edition=prelim) (CRA) “lookback window” refers to the period starting [60 working days](https://crsreports.congress.gov/product/pdf/R/R46690#page=8) (either session days in the Senate or legislative days in the House of Representatives) before the current session of Congress adjourns and ending the day the subsequent session of Congress first convenes. 
+            This dashboard allows users to explore how different [Congressional Review Act](https://uscode.house.gov/view.xhtml?req=granuleid%3AUSC-prelim-title5-chapter8&saved=%7CKHRpdGxlOjUgc2VjdGlvbjo4MDEgZWRpdGlvbjpwcmVsaW0pIE9SIChncmFudWxlaWQ6VVNDLXByZWxpbS10aXRsZTUtc2VjdGlvbjgwMSk%3D%7CdHJlZXNvcnQ%3D%7C%7C0%7Cfalse%7Cprelim&edition=prelim) (CRA) lookback window dates would affect the set of rules available for congressional review and tracks resolutions for disapproval targeting the rules published within this window. 
+            The “lookback window” refers to the period starting [60 working days](https://crsreports.congress.gov/product/pdf/R/R46690#page=8) (either session days in the Senate or legislative days in the House of Representatives) before the current session of Congress adjourns and ending the day the subsequent session of Congress first convenes. 
             Rules that are published in the Federal Register and submitted to Congress after the lookback day are made available for review in the subsequent session of Congress. 
             The current lookback date is [August 16, 2024](https://www.congress.gov/congressional-record/volume-171/issue-18/house-section/article/H398-8). 
-            This dashboard allows users to explore how different lookback window dates would affect the set of rules available for congressional review. 
             
             "Section 3(f)(1) significant" rules are regulations that meet the criteria in Section 3(f)(1) of [Executive Order 12866](https://www.archives.gov/files/federal-register/executive-orders/pdf/12866.pdf), as amended by [Executive Order 14094](https://www.govinfo.gov/content/pkg/FR-2023-04-11/pdf/2023-07760.pdf), referring to those with an estimated annual effect on the economy of $200 million or more.
             "Other significant" rules are regulations that meet the other criteria in Section 3(f) of Executive Order 12866, as amended by Executive Order 14094, such as those creating inconsistency with other agencies' actions, altering certain budgetary impacts, or raising legal or policy issues pertaining to the president's priorities.
@@ -309,7 +314,7 @@ def filtered_df(agency_column: str = "parent_slug"):
     filt_df = DF
 
     # merge with RD data
-    filt_df=merge_rd(filt_df)
+    filt_df = get_rd(filt_df)
 
     # filter dates
     try:
@@ -322,7 +327,16 @@ def filtered_df(agency_column: str = "parent_slug"):
     if (input.menu_agency() is not None) and ("all" not in input.menu_agency()):
         bool_agency = [True if sum(selected in agency for selected in input.menu_agency()) > 0 else False for agency in filt_df[agency_column]]
         filt_df = filt_df.loc[bool_agency]
-    
+
+    # filter CRA target status
+    bool_cra = []
+    if (input.menu_cra_target() is not None) and ("all" not in input.menu_cra_target()):
+        if "CRA targeted" in input.menu_cra_target():
+            bool_cra.append((filt_df["CRA Target"] == 1).to_numpy())
+        if "Not CRA targeted" in input.menu_cra_target():
+            bool_cra.append((filt_df["CRA Target"] == 0).to_numpy())
+        filt_df = filt_df.loc[array(bool_cra).any(axis=0)]
+
     # return filtered dataframe
     return filt_df
 
@@ -392,7 +406,7 @@ def grouped_df_week():
 
 @reactive.calc
 def grouped_df_agency():
-    filt_df = filter_significance()
+    filt_df=filter_significance()
     grouped = groupby_agency(filt_df, metadata=METADATA, significant=GET_SIGNIFICANT)
     return grouped
 

modules/get_resolutions.py

@@ -1,13 +1,15 @@
 import pandas as pd
+import subprocess
+import datetime
 
-# Define RD stages
-rd_stages={0:'Introduced',
-           1:'Passed one chamber',
-           2:'Passed two chambers',
-           3:'Became law',
-           4:'Vetoed by president'}
+#%% Define RD stages
+rd_stages={0:'RD Introduced',
+           1:'RD Passed One Chamber',
+           2:'RD Passed Two Chambers',
+           3:'RD Became Law',
+           4:'RD Vetoed by President'}
 
-# Function to read and clean CRA RD data set
+#%% Function to read and clean CRA RD dataset
 def read_cra_data(file_path):
     
     # Import CRA data set
@@ -16,6 +18,9 @@ def read_cra_data(file_path):
     # Refine dataframe
     df_rd=df_rd[['rd','introdate','rd_link','ridentifier','rin','introducedbin','chamberpass_atleast1','chamberpass_2','becamelawbin','vetobin']]
 
+    # Get the latest RD intro date
+    last_rd_date=max(df_rd['introdate'].astype('datetime64[ns]'))
+
     # Define RD status variable
     df_rd['rd_status']=df_rd[['chamberpass_atleast1','chamberpass_2','becamelawbin','vetobin']].sum(axis=1)
 
@@ -30,27 +35,59 @@ def read_cra_data(file_path):
 
     # Rename columns
     df_rd.rename(columns={'ridentifier':'citation','introducedbin':'CRA Target',
-                          'rd_status':'CRA Stage','rd_date': 'RD Ref & Date','rd_link':'RD Link'},inplace=True)
+                          'rd_status':'CRA Stage','rd_date': 'RD No. & Date','rd_link':'RD Link'},inplace=True)
     
     # Replace RD stage values
     df_rd['CRA Stage'] = df_rd['CRA Stage'].replace(rd_stages)
 
-    return df_rd
+    return df_rd, last_rd_date
+
+#%% Function to get last modified date of a file
+def get_last_modified_date(file_path):
+    """
+    Retrieves the last modified date of a file in Git.
+
+    Args:
+        file_path (str): The path to the file.
 
-#%% Merge with FR data set
-def merge_rd(df_fr):
+    Returns:
+        datetime: A datetime object representing the last modified date, or None if an error occurs.
+    """
+    try:
+        # Execute git log command to get the latest commit information for the file
+        result = subprocess.run(['git', 'log', '-n', '1', '--pretty=format:%at', '--', file_path], capture_output=True, text=True, check=True)
+        timestamp = int(result.stdout.strip())
+        return datetime.datetime.fromtimestamp(timestamp).strftime("%Y-%m-%d")
+    except subprocess.CalledProcessError as e:
+         print(f"Error: {e}")
+         return None
+    except ValueError:
+        print(f"Error: Could not parse timestamp from git log output.")
+        return None
+
+#%% Function to merge RD data with FR data
+def get_rd(df_fr):
     # Read CRA data set
     cra_file_path='congress_data\cra_data.csv'
-    df_rd=read_cra_data(cra_file_path)
-    # print(df_rd.head())
+    df_rd,_=read_cra_data(cra_file_path)
 
     # Merge
     df_merged=df_fr.merge(df_rd,on='citation',how='left')
 
-    # Fill in null
+    # Fill null
     df_merged['CRA Target']=df_merged['CRA Target'].fillna(0)
 
     return df_merged
 
-#%%
-# merge_rd()
+#%% Function to get last modified date of the RD dataset
+def get_cra_updated_date(cra_file_path='congress_data\cra_data.csv'):
+    cra_last_updated = get_last_modified_date(cra_file_path)
+
+    if cra_last_updated:
+        return cra_last_updated
+    else:
+        _,last_rd_date=read_cra_data(cra_file_path)
+        return last_rd_date.strftime("%Y-%m-%d")
+    
+#%% Create objects to import in app
+CRA_LAST_UPDATED=get_cra_updated_date()