Repository: Analyticsphere/qaqc_testing
Pull request #158: Stg
Author: login: KELSEYDOWLING7
Created: 2025-03-24T17:24:18Z | Merged: 2025-04-03T16:00:51Z

All changes from my local repo have been merged into the stg branch (with the new merge process we discussed last week). This is ready for your review to now be merged into prod to begin automation updates and newly automated reports 


Diff:
diff --git a/Biospecimen Metrics QC and Accession IDs.Rmd b/Biospecimen Metrics QC and Accession IDs.Rmd
new file mode 100644
index 0000000..dc1ca64
--- /dev/null
+++ b/Biospecimen Metrics QC and Accession IDs.Rmd	
@@ -0,0 +1,1021 @@
+---
+title: "Biospecimen Metrics QC and Accession IDs"
+author: "Kelsey Dowling"
+date: "`r Sys.Date()`"
+header-includes:  
+    \usepackage{placeins}
+
+
+output:
+  pdf_document:
+    extra_dependencies: ["float"]
+    toc: true
+    toc_depth: 2
+    keep_tex: yes
+    fig_width: 7
+    fig_height: 5
+    fig_caption: true
+    latex_engine: xelatex
+    df_print: paged 
+---
+
+
+
+```{r, include=FALSE}
+
+
+library(bigrquery)
+library(lubridate)
+library(dplyr)
+library(stringr)
+library(gt)
+library(expss)
+library(knitr)
+library(kableExtra)
+bq_auth()
+2
+
+project <- "nih-nci-dceg-connect-prod-6d04"
+
+# bio <- "SELECT  Connect_ID, d_232343615_d_593843561, d_143615646_d_593843561, d_223999569_d_593843561,  d_299553921_d_593843561, d_376960806_d_593843561, d_454453939_d_593843561, d_589588440_d_593843561, d_652357376_d_593843561, d_677469051_d_593843561, d_683613884_d_593843561, d_703954371_d_593843561, d_787237543_d_593843561, d_838567176_d_593843561, d_958646668_d_593843561, d_973670172_d_593843561, d_650516960, d_678166505,  d_143615646_d_926457119, d_232343615_d_926457119, d_299553921_d_926457119, d_376960806_d_926457119, d_454453939_d_926457119, d_589588440_d_926457119, d_652357376_d_926457119, d_677469051_d_926457119, d_683613884_d_926457119, d_703954371_d_926457119, d_838567176_d_926457119, d_958646668_d_926457119, d_973670172_d_926457119, d_143615646_d_883732523, d_299553921_d_883732523, d_454453939_d_883732523, d_652357376_d_883732523, d_703954371_d_883732523, d_838567176_d_883732523, d_973670172_d_883732523, d_143615646_d_593843561, d_223999569_d_593843561, d_299553921_d_593843561, d_376960806_d_593843561, d_454453939_d_593843561, d_589588440_d_593843561, d_652357376_d_593843561, d_677469051_d_593843561, d_683613884_d_593843561, d_703954371_d_593843561, d_787237543_d_593843561, d_838567176_d_593843561, d_958646668_d_593843561, d_973670172_d_593843561, d_915838974, d_646899796, 
+# d_928693120_integer FROM `nih-nci-dceg-connect-prod-6d04.FlatConnect.biospecimen_JP`  where Connect_ID IS NOT NULL"
+
+
+bio <- "WITH T AS (
+  SELECT Connect_ID FROM `nih-nci-dceg-connect-prod-6d04.FlatConnect.biospecimen_JP`
+  GROUP BY Connect_ID
+  HAVING COUNT(Connect_ID) =1) 
+SELECT * FROM `nih-nci-dceg-connect-prod-6d04.FlatConnect.biospecimen_JP`
+INNER JOIN T ON `nih-nci-dceg-connect-prod-6d04.FlatConnect.biospecimen_JP`.Connect_ID = T.Connect_ID ;"
+
+
+
+parts <- "SELECT Connect_ID, d_173836415_d_266600170_d_139245758, d_173836415_d_266600170_d_156605577, d_173836415_d_266600170_d_184451682, d_173836415_d_266600170_d_185243482, d_173836415_d_266600170_d_198261154, d_173836415_d_266600170_d_210921343, d_173836415_d_266600170_d_224596428, d_173836415_d_266600170_d_316824786, d_173836415_d_266600170_d_341570479, d_173836415_d_266600170_d_398645039, d_173836415_d_266600170_d_448660695, d_173836415_d_266600170_d_452847912, d_173836415_d_266600170_d_453452655, d_173836415_d_266600170_d_530173840, d_173836415_d_266600170_d_534041351, d_173836415_d_266600170_d_541311218, d_173836415_d_266600170_d_561681068, d_173836415_d_266600170_d_592099155, d_173836415_d_266600170_d_693370086, d_173836415_d_266600170_d_718172863, d_173836415_d_266600170_d_728696253, d_173836415_d_266600170_d_740582332, d_173836415_d_266600170_d_769615780, d_173836415_d_266600170_d_786930107, d_173836415_d_266600170_d_822274939, d_173836415_d_266600170_d_847159717, d_173836415_d_266600170_d_860477844, d_173836415_d_266600170_d_880794013, d_173836415_d_266600170_d_915179629, d_173836415_d_266600170_d_939818935, d_173836415_d_266600170_d_982213346, d_684635302, d_254109640, d_878865966, d_167958071, d_827220437, d_173836415_d_266600170_d_543608829, d_173836415_d_266600170_d_110349197
+FROM `nih-nci-dceg-connect-prod-6d04.FlatConnect.participants_JP` where Connect_ID IS NOT NULL and d_831041022='104430631'"
+
+parts_table <- bq_project_query(project, parts)
+parts_data <- bq_table_download(parts_table, bigint = "integer64")
+
+bio_table <- bq_project_query(project, bio)
+bio_data <- bq_table_download(bio_table, bigint = "integer64")
+
+bio_data$Connect_ID <- as.numeric(bio_data$Connect_ID)
+parts_data$Connect_ID <- as.numeric(parts_data$Connect_ID)
+
+
+
+#bioqc_join= left_join(parts_data, bio_data, by="Connect_ID") 
+
+bioqc= left_join(bio_data, parts_data, by=c("Connect_ID", "d_827220437") )
+
+knitr::opts_chunk$set(comment = NA)
+
+
+
+
+
+bioqc <- bioqc %>%  mutate(Site=case_when(d_827220437==531629870 ~ "HealthPartners",
+                                                 d_827220437==548392715 ~ "Henry Ford Health System",
+                                                 d_827220437==303349821 ~ "Marshfield Clinical Health System",
+                                                 d_827220437==657167265 ~ "Sanford Health",
+                                                 d_827220437==809703864 ~ "University of Chicago Medicine",
+                                                d_827220437==125001209 ~ "Kaiser Permanente Colorado",
+                                                 d_827220437==327912200 ~ "Kaiser Permanente Georgia",
+                                                 d_827220437==300267574 ~ "Kaiser Permanente Hawaii",
+                                                 d_827220437==452412599 ~ "Kaiser Permanente Northwest"))
+
+
+
+
+
+
+
+
+
+
+
+##### Making sure personal C drives aren't referenced if this code is being used by others
+
+
+#Change to FALSE if referencing this code for Box outputs
+write_to_local_drive = T #F
+
+
+### This function below determines whether the file will be created locally or not.
+local_drive= ifelse(write_to_local_drive, "C:/Users/dowlingk2/Documents/Module-Missingness-and-Metrics/data/", "")
+
+
+
+
+
+
+
+```
+
+\newpage
+
+
+```{r, echo=FALSE, warning=FALSE, message=FALSE}
+# Accession ID Comparison: KP vs Biospecimen Dashboard
+## Blood
+
+#### IN A SEPARATE REPORT
+
+
+### HAWAII ONLY: They're sending one extra number
+
+qc1 <- bioqc %>%  filter(d_827220437==300267574)
+
+#Blood
+qc1_blood <- qc1 %>%  filter(str_sub(d_173836415_d_266600170_d_341570479,1,11)!=str_sub(d_646899796_integer,1,11)) 
+
+qc1_blood <- qc1_blood %>% select(Site, Connect_ID, d_173836415_d_266600170_d_341570479, d_646899796_integer ) %>%  sort_by(Site)
+
+
+# qc1_blood %>%  gt() %>%  
+#   tab_header(title = md("KPHI Blood Accession ID Errors")) %>% 
+#   cols_label(Connect_ID = md("**Connect ID**"), d_173836415_d_266600170_d_341570479 = md("**KPHI Accession ID**"), d_646899796= md("**Expected Accession ID**"))
+
+colnames(qc1_blood) <- c("Site", "Connect ID", "KP Accession ID" , "Expected Accession ID")
+#knitr::kable(qc1_blood)
+
+
+
+#### IN A SEPARATE REPORT
+
+
+
+
+
+#Urine
+qc1_urine <- qc1 %>%  filter(str_sub(qc1$d_173836415_d_266600170_d_198261154,1,11)!=str_sub(qc1$d_928693120_integer,1,11)) 
+
+qc1_urine$AID<- strtoi(qc1_urine$d_928693120_integer, base = 0L)
+
+qc1_urine <- qc1_urine %>% select(Site, Connect_ID, d_173836415_d_266600170_d_198261154, d_928693120_integer ) %>%  sort_by(Site)
+
+# qc1_urine %>%  gt() %>%  
+#   tab_header(title = md("KPHI Urine Accession ID Errors")) %>% 
+#   cols_label(Connect_ID = md("**Connect ID**"), d_173836415_d_266600170_d_198261154 = md("**KPHI Accession ID**"), d_928693120_integer= md("**Expected Accession ID**"))
+
+colnames(qc1_urine) <- c("Site", "Connect ID", "KP Accession ID" , "Expected Accession ID")
+#knitr::kable(qc1_urine)
+
+```
+
+
+ 
+
+```{r, echo=FALSE, warning=FALSE, message=FALSE}
+
+# KP Accession IDs: Accession ID sent by sites not equal to accession ID entered in dashboard, for all other KP Sites
+
+#### IN A SEPARATE REPORT
+
+
+
+#### ALL OTHER KPS: They're sending the worng number entirely
+
+
+
+
+#Blood
+
+qcKP <- bioqc %>%  filter(d_827220437==125001209 | d_827220437==327912200 | d_827220437== 452412599) 
+
+qcKP_blood <- qcKP %>%  filter(d_173836415_d_266600170_d_341570479!=d_646899796_integer)  %>% select(Site, Connect_ID, d_173836415_d_266600170_d_341570479, d_646899796_integer ) %>%  sort_by(Site)
+
+colnames(qcKP_blood) <- c("Site", "Connect ID", "KP Accession ID" , "Expected Accession ID")
+
+#knitr::kable(qcKP_blood)
+
+
+
+
+#Urine
+
+qcKP_urine <- qcKP %>%  filter(d_173836415_d_266600170_d_198261154!=d_928693120_integer) %>% select(Site, Connect_ID, d_173836415_d_266600170_d_198261154, d_928693120_integer ) %>%  sort_by(Site)
+
+colnames(qcKP_urine) <- c("Site", "Connect ID", "KP Accession ID" , "Expected Accession ID")
+
+#knitr::kable(qcKP_urine)
+
+
+
+```
+
+
+
+
+
+### (Derived) Clinical DB Blood RRL Received (BioClin_DBBloodRRL_v1r0): If all blood tubes are not collected, this should be no
+```{r, echo=FALSE, warning=FALSE, message=FALSE}
+
+qc2 <- bioqc %>%  filter( d_232343615_d_593843561==104430631 & d_299553921_d_593843561==104430631 & d_376960806_d_593843561==104430631 & d_454453939_d_593843561==104430631 & d_589588440_d_593843561==104430631 & d_958646668_d_593843561==104430631 & d_677469051_d_593843561==104430631 & d_683613884_d_593843561==104430631 & d_703954371_d_593843561==104430631 & d_838567176_d_593843561==104430631 & d_173836415_d_266600170_d_534041351!=104430631) 
+#qc2 <- qc2  %>% select(Site, Connect_ID, d_173836415_d_266600170_d_534041351)  %>%  group_by(Site)
+qc2 <- qc2  %>% select(Site, Connect_ID, d_173836415_d_266600170_d_534041351)  %>%  sort_by(Site)
+
+
+colnames(qc2) <- c("Site", "Connect ID", "Value")
+knitr::kable(qc2)
+```
+
+
+
+### If all blood tubes are not collected, (Derived) Clinical Site Blood Collected (BioClin_SiteBloodColl_v1r0) should be no or NULL.
+```{r, echo=FALSE, warning=FALSE, message=FALSE}
+
+qc2_1 <- bioqc %>%  filter( d_232343615_d_593843561==104430631 & d_299553921_d_593843561==104430631 & d_376960806_d_593843561==104430631 & d_454453939_d_593843561==104430631 & d_589588440_d_593843561==104430631 & d_958646668_d_593843561==104430631 & d_677469051_d_593843561==104430631 & d_683613884_d_593843561==104430631 & d_703954371_d_593843561==104430631 & d_838567176_d_593843561==104430631 & !is.na(d_173836415_d_266600170_d_693370086)) 
+qc2_1 <- qc2_1  %>% select(Site, Connect_ID, d_173836415_d_266600170_d_534041351)  %>%  sort_by(Site)
+
+# qc2_1 %>%  gt() %>%  
+#   tab_header(title = md("Dervied Clinical DB Blood RRL Received (Supposed to be No")) %>% 
+#   cols_label(Connect_ID = md("**Connect ID**"), d_173836415_d_266600170_d_534041351 = md("**Value**"))
+
+colnames(qc2_1) <- c("Site", "Connect ID", "Value")
+knitr::kable(qc2_1)
+
+
+```
+
+
+
+### (Derived) Baseline blood sample collected (BioFin_BaseBloodCol_v1r0): If all blood tubes are not collected, this should be no
+```{r, echo=FALSE, warning=FALSE, message=FALSE}
+
+qc2_2 <- bioqc %>%  filter( d_232343615_d_593843561==104430631 & d_299553921_d_593843561==104430631 & d_376960806_d_593843561==104430631 & d_454453939_d_593843561==104430631 & d_589588440_d_593843561==104430631 & d_958646668_d_593843561==104430631 & d_677469051_d_593843561==104430631 & d_683613884_d_593843561==104430631 & d_703954371_d_593843561==104430631 & d_838567176_d_593843561==104430631 & d_878865966!=104430631 & 
+                              Connect_ID!=2310991421) #HP sent clinical sample when it should've only been research- needs to be manually removed
+qc2_2 <- qc2_2  %>% select(Site, Connect_ID, d_173836415_d_266600170_d_534041351)  %>%  sort_by(Site)
+
+# qc2_2 %>%  gt() %>%  
+#   tab_header(title = md("Dervied Clinical DB Blood RRL Received (Supposed to be No")) %>% 
+#   cols_label(Connect_ID = md("**Connect ID**"), d_173836415_d_266600170_d_534041351 = md("**Value**"))
+
+colnames(qc2_2) <- c("Site", "Connect ID", "Value")
+knitr::kable(qc2_2)
+
+```
+
+
+
+
+### If BioClin_BldOrderPlacdDt_v1r0 occured before BioClin_UrnOrderPlacdDt_v1r0, then BioClin_BldUrnPlacedTm_v1r0 should = BioClin_BldOrderPlacdDt_v1r0
+```{r, echo=FALSE, warning=FALSE, message=FALSE}
+
+qc3_1 <- bioqc %>%  filter(as.POSIXct(ymd_hms(d_173836415_d_266600170_d_769615780)) < as.POSIXct(ymd_hms(d_173836415_d_266600170_d_939818935)) & as.POSIXct(ymd_hms(d_173836415_d_266600170_d_184451682))!=as.POSIXct(ymd_hms(d_173836415_d_266600170_d_769615780))) %>%  select(Site, Connect_ID, d_173836415_d_266600170_d_184451682) %>%  sort_by(Site)
+
+
+# qc3_1 %>%  gt() %>%  
+#   tab_header(title = md("Date/Time first baseline blood order placed")) %>% 
+#   cols_label(Connect_ID = md("**Connect ID**"), d_173836415_d_266600170_d_184451682 = md("**Value**"))
+
+colnames(qc3_1) <- c("Site", "Connect ID", "Value")
+knitr::kable(qc3_1)
+```
+
+
+### If BioClin_BldOrderPlacdDt_v1r0 occured after BioClin_UrnOrderPlacdDt_v1r0, then BioClin_BldUrnPlacedTm_v1r0 should = BioClin_UrnOrderPlacdDt_v1r0
+```{r, echo=FALSE, warning=FALSE, message=FALSE}
+qc3_2 <- bioqc %>%  filter(as.POSIXct(ymd_hms(d_173836415_d_266600170_d_769615780)) > as.POSIXct(ymd_hms(d_173836415_d_266600170_d_939818935)) & as.POSIXct(ymd_hms(d_173836415_d_266600170_d_184451682))!=as.POSIXct(ymd_hms(d_173836415_d_266600170_d_939818935))) %>% select(Site, Connect_ID, d_173836415_d_266600170_d_184451682) %>%  sort_by(Site)
+
+
+# qc3_2 %>%  gt() %>%  
+#   tab_header(title = md("Date/Time first baseline urine order placed")) %>% 
+#   cols_label(Connect_ID = md("**Connect ID**"), d_173836415_d_266600170_d_184451682 = md("**Value**"))
+
+colnames(qc3_2) <- c("Site", "Connect ID", "Value")
+knitr::kable(qc3_2)
+
+```
+
+
+
+###	If the Urine Tube was collected and BioClin_DBUrineRRLDt_v1r0 occurred more than seven days ago, BioClin_SiteUrineColl_v1r0 must be yes
+```{r, echo=FALSE, warning=FALSE, message=FALSE}
+
+currentDate <- Sys.Date()
+
+qc4 <- bioqc %>%  filter(d_973670172_d_593843561==353358909 & as.numeric(round(difftime(currentDate, d_173836415_d_266600170_d_541311218, units="days"), digits=0)) > 7  & is.na(d_173836415_d_266600170_d_786930107)) %>% 
+  select(Site, Connect_ID, d_173836415_d_266600170_d_786930107) %>%  sort_by(Site)
+
+# qc4 %>%  gt() %>%  
+#   tab_header(title = md("Site Clinical Urine Collected (Supposed to be Yes")) %>% 
+#   cols_label(Connect_ID = md("**Connect ID**"), d_173836415_d_266600170_d_786930107 = md("**Value**"))
+
+colnames(qc4) <- c("Site", "Connect ID", "Value")
+knitr::kable(qc4)
+
+```
+
+
+### If(Site Serum Separator Tube Number) has been collected, the collection site is clinical, and BioClin_DBBloodRRLDt_v1r0 occured more than 7 days ago, then BioClin_SiteBloodColl_v1r0 should be yes
+```{r, echo=FALSE, warning=FALSE, message=FALSE}
+
+qc5_tube4 <- bioqc %>%  filter( d_232343615_d_593843561==353358909 & d_650516960==664882224 & as.numeric(round(difftime(currentDate, d_173836415_d_266600170_d_398645039, units="days"), digits=0)) > 7  & d_173836415_d_266600170_d_693370086 !=353358909) %>% 
+  select(Site, Connect_ID, d_173836415_d_266600170_d_693370086) %>%  sort_by(Site)
+
+# qc5_tube4 %>%  gt() %>%  
+#   tab_header(title = md("Site Serum Separator Tube 4 Collected (Supposed to be Yes)")) %>% 
+#   cols_label(Connect_ID = md("**Connect ID**"), d_173836415_d_266600170_d_693370086 = md("**Value**"))
+
+colnames(qc5_tube4) <- c("Site", "Connect ID", "Site Serum Separator Tube 4 Collected (Supposed to be Yes)")
+knitr::kable(qc5_tube4)
+
+
+
+
+
+qc5_tube1 <- bioqc %>%  filter( d_299553921_d_593843561==353358909 & d_650516960==664882224 & as.numeric(round(difftime(currentDate, d_173836415_d_266600170_d_398645039, units="days"), digits=0)) > 7  & d_173836415_d_266600170_d_693370086 !=353358909) %>% 
+  select(Site, Connect_ID, d_173836415_d_266600170_d_693370086) %>%  sort_by(Site)
+
+# qc5_tube1 %>%  gt() %>%  
+#   tab_header(title = md("Site Serum Separator Tube 1 Collected (Supposed to be Yes)")) %>% 
+#   cols_label(Connect_ID = md("**Connect ID**"), d_173836415_d_266600170_d_693370086 = md("**Value**"))
+
+colnames(qc5_tube1) <- c("Site", "Connect ID", "Site Serum Separator Tube 1 Collected (Supposed to be Yes)")
+knitr::kable(qc5_tube1)
+
+
+
+
+qc5_tube3 <- bioqc %>%  filter( d_376960806_d_593843561==353358909 & d_650516960==664882224 & as.numeric(round(difftime(currentDate, d_173836415_d_266600170_d_398645039, units="days"), digits=0)) > 7  & d_173836415_d_266600170_d_693370086 !=353358909) %>% 
+  select(Site, Connect_ID, d_173836415_d_266600170_d_693370086) %>%  sort_by(Site)
+
+# qc5_tube3 %>%  gt() %>%  
+#   tab_header(title = md("Site Serum Separator Tube 3 Collected (Supposed to be Yes)")) %>% 
+#   cols_label(Connect_ID = md("**Connect ID**"), d_173836415_d_266600170_d_693370086 = md("**Value**"))
+
+colnames(qc5_tube3) <- c("Site", "Connect ID", "Site Serum Separator Tube 3 Collected (Supposed to be Yes)")
+knitr::kable(qc5_tube3)
+
+
+
+
+qc5_tube5 <- bioqc %>%  filter( d_589588440_d_593843561==353358909 & d_650516960==664882224 & as.numeric(round(difftime(currentDate, d_173836415_d_266600170_d_398645039, units="days"), digits=0)) > 7  & d_173836415_d_266600170_d_693370086 !=353358909) %>% 
+  select(Site, Connect_ID, d_173836415_d_266600170_d_693370086) %>%  sort_by(Site)
+
+# qc5_tube5 %>%  gt() %>%  
+#   tab_header(title = md("Site Serum Separator Tube 5 Collected (Supposed to be Yes)")) %>% 
+#   cols_label(Connect_ID = md("**Connect ID**"), d_173836415_d_266600170_d_693370086 = md("**Value**"))
+
+colnames(qc5_tube5) <- c("Site", "Connect ID", "Site Serum Separator Tube 5 Collected (Supposed to be Yes)")
+knitr::kable(qc5_tube5)
+
+
+
+
+qc5_tube2 <- bioqc %>%  filter( d_703954371_d_593843561==353358909 & d_650516960==664882224 & as.numeric(round(difftime(currentDate, d_173836415_d_266600170_d_398645039, units="days"), digits=0)) > 7  & d_173836415_d_266600170_d_693370086 !=353358909) %>% 
+  select(Site, Connect_ID, d_173836415_d_266600170_d_693370086) %>%  sort_by(Site)
+
+# qc5_tube2 %>%  gt() %>%  
+#   tab_header(title = md("Site Serum Separator Tube 2 Collected (Supposed to be Yes)")) %>% 
+#   cols_label(Connect_ID = md("**Connect ID**"), d_173836415_d_266600170_d_693370086 = md("**Value**"))
+
+colnames(qc5_tube2) <- c("Site", "Connect ID", "Site Serum Separator Tube 2 Collected (Supposed to be Yes)")
+knitr::kable(qc5_tube2)
+
+
+```
+
+
+
+### If(EDTA Tube Number) has been collected, the collection site is clinical, and BioClin_DBBloodRRLDt_v1r0 occured more than 7 days ago, then BioClin_SiteBloodColl_v1r0 should be yes
+```{r, echo=FALSE, warning=FALSE, message=FALSE}
+
+qc6_tube1 <- bioqc %>%  filter( d_454453939_d_593843561==353358909 & d_650516960==664882224 & as.numeric(round(difftime(currentDate, d_173836415_d_266600170_d_398645039, units="days"), digits=0)) > 7  & d_173836415_d_266600170_d_693370086 !=353358909) %>% 
+  select(Site, Connect_ID, d_173836415_d_266600170_d_693370086) %>%  sort_by(Site)
+
+# qc6_tube1 %>%  gt() %>%  
+#   tab_header(title = md("Site EDTA Tube 1 Collected (Supposed to be Yes)")) %>% 
+#   cols_label(Connect_ID = md("**Connect ID**"), d_173836415_d_266600170_d_693370086 = md("**Value**"))
+
+colnames(qc6_tube1) <- c("Site", "Connect ID", "Site EDTA Tube 1 Collected (Supposed to be Yes)")
+knitr::kable(qc6_tube1)
+
+
+
+
+qc6_tube2 <- bioqc %>%  filter( d_677469051_d_593843561==353358909 & d_650516960==664882224 & as.numeric(round(difftime(currentDate, d_173836415_d_266600170_d_398645039, units="days"), digits=0)) > 7  & d_173836415_d_266600170_d_693370086 !=353358909) %>% 
+  select(Site, Connect_ID, d_173836415_d_266600170_d_693370086) %>%  sort_by(Site)
+
+# qc6_tube2 %>%  gt() %>%  
+#   tab_header(title = md("Site EDTA Tube 2 Collected (Supposed to be Yes)")) %>% 
+#   cols_label(Connect_ID = md("**Connect ID**"), d_173836415_d_266600170_d_693370086 = md("**Value**"))
+
+colnames(qc6_tube2) <- c("Site", "Connect ID", "Site EDTA Tube 2 Collected (Supposed to be Yes)")
+knitr::kable(qc6_tube2)
+
+
+
+
+qc6_tube3 <- bioqc %>%  filter( d_683613884_d_593843561==353358909 & d_650516960==664882224 & as.numeric(round(difftime(currentDate, d_173836415_d_266600170_d_398645039, units="days"), digits=0)) > 7  & d_173836415_d_266600170_d_693370086 !=353358909) %>% 
+  select(Site, Connect_ID, d_173836415_d_266600170_d_693370086) %>%  sort_by(Site)
+
+# qc6_tube3 %>%  gt() %>%  
+#   tab_header(title = md("Site EDTA Tube 3 Collected (Supposed to be Yes)")) %>% 
+#   cols_label(Connect_ID = md("**Connect ID**"), d_173836415_d_266600170_d_693370086 = md("**Value**"))
+
+colnames(qc6_tube3) <- c("Site", "Connect ID", "Site EDTA Tube 3 Collected (Supposed to be Yes)")
+knitr::kable(qc6_tube3)
+
+
+
+```
+
+
+###  If Heparin Tube 1 was collected, site was Clinical, and BioClin_DBBloodRRLDt_v1r0 occurred more than 7 days ago, BioClin_SiteBloodColl_v1r0 must be yes
+
+```{r, echo=FALSE, warning=FALSE, message=FALSE}
+
+qc7_tube1 <- bioqc %>%  filter( d_838567176_d_593843561==353358909 & d_650516960==664882224 & as.numeric(round(difftime(currentDate, d_173836415_d_266600170_d_398645039, units="days"), digits=0)) > 7  & is.na(d_173836415_d_266600170_d_693370086)) %>% 
+  select(Site, Connect_ID, d_173836415_d_266600170_d_693370086) %>%  sort_by(Site)
+
+# qc7_tube1 %>%  gt() %>%  
+#   tab_header(title = md("Site Heparin Tube 1 Collected (Supposed to be Yes")) %>% 
+#   cols_label(Connect_ID = md("**Connect ID**"), d_173836415_d_266600170_d_693370086 = md("**Value**"))
+
+colnames(qc7_tube1) <- c("Site", "Connect ID", "Site Heparin Tube 1 Collected (Supposed to be Yes)")
+knitr::kable(qc7_tube1)
+```
+
+### If Heparin Tube 2 was collected, site was Clinical, and BioClin_DBBloodRRLDt_v1r0 occurred more than 7 days ago, BioClin_SiteBloodColl_v1r0 must be yes
+```{r, echo=FALSE, warning=FALSE, message=FALSE}
+qc7_tube2 <- bioqc %>%  filter( d_958646668_d_593843561==353358909 & d_650516960==664882224 & as.numeric(round(difftime(currentDate, d_173836415_d_266600170_d_398645039, units="days"), digits=0)) > 7  & is.na(d_173836415_d_266600170_d_693370086)) %>% 
+  select(Site, Connect_ID, d_173836415_d_266600170_d_693370086) %>%  sort_by(Site)
+
+# qc7_tube2 %>%  gt() %>%  
+#   tab_header(title = md("Site Heparin Tube 2 Collected (Supposed to be Yes")) %>% 
+#   cols_label(Connect_ID = md("**Connect ID**"), d_173836415_d_266600170_d_693370086 = md("**Value**"))
+
+colnames(qc7_tube2) <- c("Site", "Connect ID", "Site Heparin Tube 2 Collected (Supposed to be Yes)")
+knitr::kable(qc7_tube2)
+
+```
+
+
+
+### All baseline samples ("BioFin_BaseBloodCol_v1r0","BioFin_BaseUrineCol_v1r0","BioFin_BaseMWCol_v1r0") are collected, and BioCol_ColTime_v1r0 occured on or after 11/21/2022 then SMMet_BLSamplesColl_v1r0 must be YES
+```{r, echo=FALSE, warning=FALSE, message=FALSE}
+
+qc8 <- bioqc %>%  filter(d_878865966==353358909 & d_167958071==353358909 & d_684635302==353358909 & d_678166505>=as.Date("2022-11-21")  & d_254109640 !=353358909) %>% 
+  select(Site, Connect_ID, d_254109640) %>%  sort_by(Site)
+
+# qc8 %>%  gt() %>%  
+#   tab_header(title = md("All baseline samples collected (Supposed to be Yes)")) %>% 
+#   cols_label(Connect_ID = md("**Connect ID**"), d_254109640 = md("**Value**"))
+
+colnames(qc8) <- c("Site", "Connect ID", "All baseline samples collected (Supposed to be Yes)")
+knitr::kable(qc8)
+
+```
+
+
+
+### If BioFin_BaseBloodCol_v1r0 is yes and the collection setting is Research, and BioCol_ColTime_v1r0 occured on or after 11/21/2022 then (Autogenerated) Date/time Baseline Research Blood Collected must be populated  
+```{r, echo=FALSE, warning=FALSE, message=FALSE}
+
+qc9B <- bioqc %>%  filter(d_878865966==353358909 & d_650516960==534621077 & d_678166505>=as.Date("2022-11-21") & is.na(d_173836415_d_266600170_d_561681068)) %>% 
+  select(Site, Connect_ID, d_173836415_d_266600170_d_561681068) %>%  sort_by(Site)
+
+# qc9B %>%  gt() %>%  
+#   tab_header(title = md("(Autogenerated) Date/time Baseline Research Blood Collected")) %>% 
+#   cols_label(Connect_ID = md("**Connect ID**"), d_173836415_d_266600170_d_561681068 = md("**Value**"))
+
+colnames(qc9B) <- c("Site", "Connect ID", "(Autogenerated) Date/time Baseline Research Blood Collected")
+knitr::kable(qc9B)
+```
+
+### If BioFin_BaseUrineCol_v1r0 is yes and the collection setting is Research, and BioCol_ColTime_v1r0 occured on or after 11/21/2022 then (Autogenerated) Date/time Baseline Research Urine Collected must be populated
+```{r, echo=FALSE, warning=FALSE, message=FALSE}
+qc9U <- bioqc %>%  filter(d_167958071==353358909 & d_650516960==534621077  & d_678166505>=as.Date("2022-11-21") & is.na(d_173836415_d_266600170_d_847159717)) %>% 
+  select(Site, Connect_ID, d_173836415_d_266600170_d_847159717) %>%  sort_by(Site)
+
+# qc9U %>%  gt() %>%  
+#   tab_header(title = md("(Autogenerated) Date/time Baseline Research Urine Collected")) %>% 
+#   cols_label(Connect_ID = md("**Connect ID**"), d_173836415_d_266600170_d_847159717 = md("**Value**"))
+
+colnames(qc9U) <- c("Site", "Connect ID", "(Autogenerated) Date/time Baseline Research Urine Collected")
+knitr::kable(qc9U)
+
+
+
+```
+
+
+
+### If BioFin_BaseBloodCol_v1r0 is yes and the collection setting is Research, BioCol_ColTime_v1r0 must be populated 
+```{r, echo=FALSE, warning=FALSE, message=FALSE}
+
+qc10 <- bioqc %>%  filter(d_878865966==353358909 & d_650516960==534621077   & is.na(d_678166505)) %>% #& d_678166505>=as.Date("2022-11-21")) %>% 
+  select(Site, Connect_ID, d_678166505) %>%  sort_by(Site)
+
+# qc10 %>%  gt() %>%  
+#   tab_header(title = md("Collection Date/Time")) %>% 
+#   cols_label(Connect_ID = md("**Connect ID**"), d_678166505 = md("**Value**"))
+
+colnames(qc10) <- c("Site", "Connect ID", "Collection Date/Time (Supposed to be populated)")
+knitr::kable(qc10)
+
+```
+
+
+
+### If BioCol_ObjCollected_v1r0 (mouthwash), and BioCol_ColTime_v1r0 occured on or after 11/21/2022, then  BioFin_BaseMouthCol_v1r0 should be no 
+```{r, echo=FALSE, warning=FALSE, message=FALSE}
+
+qc11 <- bioqc %>%  filter(d_143615646_d_593843561==104430631  & d_678166505>=as.Date("2022-11-21") & d_684635302!=104430631) %>% 
+  select(Site, Connect_ID, d_684635302) %>%  sort_by(Site)
+
+# qc11 %>%  gt() %>%  
+#   tab_header(title = md("Baseline Mouthwash Collected B")) %>% 
+#   cols_label(Connect_ID = md("**Connect ID**"), d_684635302 = md("**Value**"))
+
+colnames(qc11) <- c("Site", "Connect ID", "Baseline Mouthwash Collected (Supposed to be no)")
+knitr::kable(qc11)
+
+```
+
+
+
+### If Urine tube was collected, site was Clinical, and BioReg_ArRegTime_v1r0 occurred more than 7 days ago, BioClin_SiteUrineColl_v1r0must be yes
+```{r, echo=FALSE, warning=FALSE, message=FALSE}
+
+
+qc12 <- bioqc %>%  filter(d_973670172_d_593843561==353358909  & d_650516960==664882224 & as.numeric(round(difftime(currentDate, d_915838974, units="days"), digits=0)) >7 & is.na(d_173836415_d_266600170_d_786930107)) %>% 
+  select(Site, Connect_ID, d_173836415_d_266600170_d_786930107) %>%  sort_by(Site)
+
+# qc12 %>%  gt() %>%  
+#   tab_header(title = md("Site Clinical_Site Urine Collected (Supposed to be Yes)")) %>% 
+#   cols_label(Connect_ID = md("**Connect ID**"), d_173836415_d_266600170_d_786930107 = md("**Value**"))
+
+colnames(qc12) <- c("Site", "Connect ID", "Site Clinical_Site Urine Collected")
+knitr::kable(qc12) 
+
+
+```
+
+
+
+### Tube Date Received - If BioClin_DBBloodRRLDt_v1r0 occured more than 4 days ago, (tube type) was collected, and (tube type) was not discarded, then BioBPTL_DateRec_v1r0 for that tube type should be populated 
+```{r, echo=FALSE, warning=FALSE, message=FALSE}
+
+
+ ###########chnage to tube collected =yes and BioBPTL_DateRec_v1r0 is na for all & deviation!= discard deviation (one flag triggered)
+
+# qc13_tube1 <- bioqc %>%  filter(as.numeric(round(difftime(currentDate, d_173836415_d_266600170_d_398645039, units="days"), digits=0))>4 & is.na(d_143615646_d_926457119)) %>% 
+#   select(Site, Connect_ID, d_143615646_d_926457119)
+# 
+# qc13_tube1 %>%  gt() %>%  
+#   tab_header(title = md("MW Tube 1- Date Received - Site Shipment")) %>% 
+#   cols_label(Connect_ID = md("**Connect ID**"), d_143615646_d_926457119 = md("**Value**"))
+
+
+
+
+qc13_tube2 <- bioqc %>%  filter(as.numeric(round(difftime(currentDate, d_173836415_d_266600170_d_398645039, units="days"), digits=0))>4 & d_299553921_d_593843561==353358909 & d_299553921_d_762124027==104430631 & is.na(d_299553921_d_926457119)) %>% 
+  select(Site, Connect_ID, d_299553921_d_926457119) %>%  sort_by(Site)
+
+# qc13_tube2 %>%  gt() %>%  
+#   tab_header(title = md("Serum Separator Tube 1- Date Received - Site Shipment")) %>% 
+#   cols_label(Connect_ID = md("**Connect ID**"), d_299553921_d_926457119 = md("**Value**"))
+
+colnames(qc13_tube2) <- c("Site", "Connect ID", "Serum Separator Tube 1- Date Received")
+knitr::kable(qc13_tube2)
+
+
+
+
+qc13_tube3 <- bioqc %>%  filter(as.numeric(round(difftime(currentDate, d_173836415_d_266600170_d_398645039, units="days"), digits=0))>4 & d_454453939_d_593843561==353358909 & d_454453939_d_762124027==104430631 & is.na(d_454453939_d_926457119)) %>% 
+  select(Site, Connect_ID, d_454453939_d_926457119) %>%  sort_by(Site)
+
+# qc13_tube3 %>%  gt() %>%  
+#   tab_header(title = md("EDTA Tube 1-- Date Received - Site Shipment")) %>% 
+#   cols_label(Connect_ID = md("**Connect ID**"), d_454453939_d_926457119 = md("**Value**"))
+
+colnames(qc13_tube3) <- c("Site", "Connect ID", "EDTA Tube 1- Date Received")
+knitr::kable(qc13_tube3)
+
+
+
+
+# qc13_tube4 <- bioqc %>%  filter(as.numeric(round(difftime(currentDate, d_173836415_d_266600170_d_398645039, units="days"), digits=0))>4 & is.na(d_652357376_d_926457119) & d_650516960==664882224) %>% 
+#   select(Site, Connect_ID, d_652357376_d_926457119) %>%  sort_by(Site)
+# 
+# # qc13_tube4 %>%  gt() %>%  
+# #   tab_header(title = md("ACD Tube 1- Date Received - Site Shipment")) %>% 
+# #   cols_label(Connect_ID = md("**Connect ID**"), d_652357376_d_926457119 = md("**Value**"))
+# 
+# colnames(qc13_tube4) <- c("Site", "Connect ID", "ACD Tube 1- Date Received") ------- THIS ONE IS NO LONGER NEEDED
+# knitr::kable(qc13_tube4)
+
+
+# qc13_tube6nw <- qc13_tube4 %>%  filter(Site=="Kaiser Permanente Northwest") 
+# qc13_tube6hi <- qc13_tube4  %>%  filter(Site=="Kaiser Permanente Hawaii") 
+# qc13_tube6co <- qc13_tube4  %>%  filter(Site=="Kaiser Permanente Colorado") 
+# qc13_tube6ga <- qc13_tube4  %>%  filter(Site=="Kaiser Permanente Georgia")
+# qc13_tube6nw[, 2]
+# qc13_tube6hi[, 2]
+# qc13_tube6ga[, 2]
+# qc13_tube6co[, 2]
+# 
+# write.csv(qc13_tube4, "KP Date Received Site Shipment")
+# write_xlsx(df, "C:\\Users\\Ron\\Desktop\\Test\\people.xlsx")
+
+
+qc13_tube5 <- bioqc %>%  filter(as.numeric(round(difftime(currentDate, d_173836415_d_266600170_d_398645039, units="days"), digits=0))>4 & d_703954371_d_593843561==353358909 & d_703954371_d_762124027==104430631 & is.na(d_703954371_d_926457119)) %>% 
+  select(Site, Connect_ID, d_703954371_d_926457119) %>%  sort_by(Site)
+
+# qc13_tube5 %>%  gt() %>%  
+#   tab_header(title = md("Serum Separator Tube 2- Date Received - Site Shipment")) %>% 
+#   cols_label(Connect_ID = md("**Connect ID**"), d_703954371_d_926457119 = md("**Value**"))
+
+colnames(qc13_tube5) <- c("Site", "Connect ID", "Serum Separator Tube 2- Date Received")
+knitr::kable(qc13_tube5)
+
+
+
+
+qc13_tube6 <- bioqc %>%  filter(as.numeric(round(difftime(currentDate, d_173836415_d_266600170_d_398645039, units="days"), digits=0))>4 & d_838567176_d_593843561==353358909 & d_838567176_d_762124027==104430631 &  is.na(d_838567176_d_926457119)) %>% 
+  select(Site, Connect_ID, d_838567176_d_926457119) %>%  sort_by(Site)
+
+# qc13_tube6 %>%  gt() %>%  
+#   tab_header(title = md("Heparin Tube 1- Date Received - Site Shipment")) %>% 
+#   cols_label(Connect_ID = md("**Connect ID**"), d_838567176_d_926457119 = md("**Value**"))
+
+colnames(qc13_tube6) <- c("Site", "Connect ID", "Heparin Tube 1- Date Received")
+knitr::kable(qc13_tube6)
+
+
+
+qc13_tube8 <- bioqc %>%  filter(as.numeric(round(difftime(currentDate, d_173836415_d_266600170_d_398645039, units="days"), digits=0))>4 & d_958646668_d_593843561==353358909 & d_958646668_d_762124027==104430631 &  is.na(d_958646668_d_926457119)) %>% 
+  select(Site, Connect_ID, d_958646668_d_926457119) %>%  sort_by(Site)
+
+# qc13_tube6 %>%  gt() %>%  
+#   tab_header(title = md("Heparin Tube 1- Date Received - Site Shipment")) %>% 
+#   cols_label(Connect_ID = md("**Connect ID**"), d_958646668_d_926457119 = md("**Value**"))
+
+colnames(qc13_tube8) <- c("Site", "Connect ID", "Heparin Tube 2- Date Received")
+knitr::kable(qc13_tube8)
+
+
+
+
+qc13_tube7 <- bioqc %>%  filter(as.numeric(round(difftime(currentDate, d_173836415_d_266600170_d_541311218, units="days"), digits=0))>4 & d_973670172_d_593843561==353358909 & d_973670172_d_762124027==104430631 & is.na(d_973670172_d_926457119)) %>% 
+  select(Site, Connect_ID, d_973670172_d_926457119) %>%  sort_by(Site)
+
+# qc13_tube7 %>%  gt() %>%  
+#   tab_header(title = md("Urine Tube 1-- Date Received - Site Shipment")) %>% 
+#   cols_label(Connect_ID = md("**Connect ID**"), d_973670172_d_926457119 = md("**Value**"))
+
+colnames(qc13_tube7) <- c("Site", "Connect ID", "Urine Tube 1- Date Received")
+knitr::kable(qc13_tube7)
+
+
+
+
+```
+
+
+
+
+
+
+
+
+
+\FloatBarrier
+
+### BioClin_DBBloodID_v1r0 must be in BioClin_PolyBloodID_v1r0
+```{r, echo=FALSE, warning=FALSE, message=FALSE}
+
+bioqc1 <- bioqc %>%  filter(d_173836415_d_266600170_d_543608829!="[]" & !is.na(d_646899796_integer)) %>%  mutate(BioClin_DBBloodID_v1r0=paste0("L", d_646899796_integer))
+
+polyblood <- bioqc1 %>%  filter(!(BioClin_DBBloodID_v1r0 %in% as.list(d_173836415_d_266600170_d_543608829))) %>%  select(Site, Connect_ID, BioClin_DBBloodID_v1r0, d_173836415_d_266600170_d_543608829) %>%  sort_by(Site)
+colnames(polyblood) <- c("Site", "Connect ID", "BioClin_DBBloodID_v1r0", "BioClin_PolyBloodID_v1r0")
+knitr::kable(polyblood) %>% kable_styling(latex_options = "scale_down")
+
+
+```
+
+\FloatBarrier
+
+### BioClin_DBUrineID_v1r0 must be in BioClin_PolyUrineID_v1r0
+```{r, echo=FALSE, warning=FALSE, message=FALSE}
+
+bioqc1 <- bioqc1 %>%  filter(d_173836415_d_266600170_d_110349197!="[]" & !is.na(d_928693120_integer)) %>%  mutate(BioClin_DBUrineID_v1r0=paste0("L", d_928693120_integer))
+
+bioqc1$BioClin_PolyUrineID_v1r0 <- str_sub(bioqc1$d_173836415_d_266600170_d_110349197, 2, -2)
+
+polyurine <- bioqc1 %>%  filter(!(BioClin_DBUrineID_v1r0 %in% as.list(BioClin_PolyUrineID_v1r0))) %>%  
+  select(Site, Connect_ID, BioClin_DBUrineID_v1r0, BioClin_PolyUrineID_v1r0) %>%  sort_by(Site)
+colnames(polyurine) <- c("Site", "Connect ID", "BioClin_DBUrineID_v1r0", "BioClin_PolyUrineID_v1r0")
+knitr::kable(polyurine) %>% kable_styling(latex_options = "scale_down")
+
+```
+
+
+
+
+
+
+
+
+
+
+```{r, echo=FALSE, warning=FALSE, message=FALSE, include=FALSE}
+
+
+### Reason Tube Not Collected: Will circle back to addressing this?
+
+qc14_tube1 <- bioqc %>%  filter(d_143615646_d_593843561==104430631 & d_650516960== 534621077 & is.na(d_143615646_d_883732523) ) %>% 
+  select(Site, Connect_ID, d_143615646_d_883732523) %>%  sort_by(Site)
+
+# qc14_tube1 %>%  gt() %>%  
+#   tab_header(title = md("MW Tube 1- Reason Tube Not Collected")) %>% 
+#   cols_label(Connect_ID = md("**Connect ID**"), d_143615646_d_883732523 = md("**Value**"))
+
+# colnames(qc14_tube1) <- c("Site", "Connect ID", "MW Tube 1- Reason Tube Not Collected")
+#knitr::kable( qc14_tube1)
+
+
+
+qc14_tube2 <- bioqc %>%  filter(d_299553921_d_593843561==104430631 & d_650516960== 534621077 & is.na(d_299553921_d_883732523)) %>% 
+  select(Site, Connect_ID, d_299553921_d_883732523) %>%  sort_by(Site)
+
+# qc14_tube2 %>%  gt() %>%  
+#   tab_header(title = md("Serum Separator Tube 1- Reason Tube Not Collected")) %>% 
+#   cols_label(Connect_ID = md("**Connect ID**"), d_299553921_d_883732523 = md("**Value**"))
+
+colnames(qc14_tube2) <- c("Site", "Connect ID", "Serum Separator Tube 1- Reason Tube Not Collected")
+#knitr::kable(qc14_tube2)
+
+
+
+
+qc14_tube3 <- bioqc %>%  filter(d_454453939_d_593843561==104430631 & d_650516960== 534621077 & is.na(d_454453939_d_883732523)) %>% 
+  select(Site, Connect_ID, d_454453939_d_883732523) %>%  sort_by(Site)
+
+# qc14_tube3 %>%  gt() %>%  
+#   tab_header(title = md("EDTA Tube 1- Reason Tube Not Collected")) %>% 
+#   cols_label(Connect_ID = md("**Connect ID**"), d_454453939_d_883732523 = md("**Value**"))
+
+colnames(qc14_tube3) <- c("Site", "Connect ID", "EDTA Tube 1- Reason Tube Not Collected")
+#knitr::kable(qc14_tube3)
+
+
+
+
+qc14_tube4 <- bioqc %>%  filter(d_652357376_d_593843561==104430631 & d_650516960== 534621077 & is.na(d_652357376_d_883732523)) %>% 
+  select(Site, Connect_ID, d_652357376_d_883732523) %>%  sort_by(Site)
+
+# qc14_tube4 %>%  gt() %>%  
+#   tab_header(title = md("ACD Tube 1- Reason Tube Not Collected")) %>% 
+#   cols_label(Connect_ID = md("**Connect ID**"), d_652357376_d_883732523 = md("**Value**"))
+
+colnames(qc14_tube4) <- c("Site", "Connect ID", "ACD Tube 1- Reason Tube Not Collected")
+#knitr::kable(qc14_tube4)
+
+
+
+qc14_tube5 <- bioqc %>%  filter(d_703954371_d_593843561==104430631 & d_650516960== 534621077 & is.na(d_703954371_d_883732523)) %>% 
+  select(Site, Connect_ID, d_703954371_d_883732523) %>%  sort_by(Site)
+
+# qc14_tube5 %>%  gt() %>%  
+#   tab_header(title = md("Serum Separator Tube 2-- Reason Tube Not Collected")) %>% 
+#   cols_label(Connect_ID = md("**Connect ID**"), d_703954371_d_883732523 = md("**Value**"))
+
+colnames(qc14_tube5) <- c("Site", "Connect ID", "Serum Separator Tube 2- Reason Tube Not Collected")
+#knitr::kable(qc14_tube5)
+
+
+
+qc14_tube6 <- bioqc %>%  filter(d_838567176_d_593843561==104430631 & d_650516960== 534621077 & is.na(d_838567176_d_883732523)) %>% 
+  select(Site, Connect_ID, d_838567176_d_883732523) %>%  sort_by(Site)
+
+# qc14_tube6 %>%  gt() %>%  
+#   tab_header(title = md("Heparin Tube 1- Reason Tube Not Collected")) %>% 
+#   cols_label(Connect_ID = md("**Connect ID**"), d_838567176_d_883732523 = md("**Value**"))
+
+colnames(qc14_tube6) <- c("Site", "Connect ID", "Heparin Tube 1- Reason Tube Not Collected")
+#knitr::kable(qc14_tube6)
+
+
+
+qc14_tube7 <- bioqc %>%  filter(d_973670172_d_593843561==104430631 & d_650516960== 534621077 & is.na(d_973670172_d_883732523)) %>% 
+  select(Site, Connect_ID, d_973670172_d_883732523) %>%  sort_by(Site)
+
+# qc14_tube7 %>%  gt() %>%  
+#   tab_header(title = md("Urine Tube 1- Reason Tube Not Collected")) %>% 
+#   cols_label(Connect_ID = md("**Connect ID**"), d_973670172_d_883732523 = md("**Value**"))
+
+colnames(qc14_tube7) <- c("Site", "Connect ID", "Urine Tube 1- Reason Tube Not Collected")
+#knitr::kable(qc14_tube7)
+
+```
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+```{r, include=FALSE}
+
+
+
+bio <- "WITH T AS (
+  SELECT Connect_ID FROM `nih-nci-dceg-connect-prod-6d04.FlatConnect.biospecimen_JP`
+  GROUP BY Connect_ID
+  HAVING COUNT(Connect_ID) =1 ) 
+SELECT * FROM `nih-nci-dceg-connect-prod-6d04.FlatConnect.biospecimen_JP`
+INNER JOIN T ON `nih-nci-dceg-connect-prod-6d04.FlatConnect.biospecimen_JP`.Connect_ID = T.Connect_ID ;"
+
+
+
+parts <- "SELECT Connect_ID, d_173836415_d_266600170_d_139245758, d_173836415_d_266600170_d_156605577, d_173836415_d_266600170_d_184451682, d_173836415_d_266600170_d_185243482, d_173836415_d_266600170_d_198261154, d_173836415_d_266600170_d_210921343, d_173836415_d_266600170_d_224596428, d_173836415_d_266600170_d_316824786, d_173836415_d_266600170_d_341570479, d_173836415_d_266600170_d_398645039, d_173836415_d_266600170_d_448660695, d_173836415_d_266600170_d_452847912, d_173836415_d_266600170_d_453452655, d_173836415_d_266600170_d_530173840, d_173836415_d_266600170_d_534041351, d_173836415_d_266600170_d_541311218, d_173836415_d_266600170_d_561681068, d_173836415_d_266600170_d_592099155, d_173836415_d_266600170_d_693370086, d_173836415_d_266600170_d_718172863, d_173836415_d_266600170_d_728696253, d_173836415_d_266600170_d_740582332, d_173836415_d_266600170_d_769615780, d_173836415_d_266600170_d_786930107, d_173836415_d_266600170_d_822274939, d_173836415_d_266600170_d_847159717, d_173836415_d_266600170_d_860477844, d_173836415_d_266600170_d_880794013, d_173836415_d_266600170_d_915179629, d_173836415_d_266600170_d_939818935, d_173836415_d_266600170_d_982213346, d_684635302, d_254109640, d_878865966, d_167958071, d_827220437
+FROM `nih-nci-dceg-connect-prod-6d04.FlatConnect.participants_JP` where Connect_ID IS NOT NULL"
+
+parts_table <- bq_project_query(project, parts)
+parts_data <- bq_table_download(parts_table, bigint = "integer64")
+
+bio_table <- bq_project_query(project, bio)
+bio_data <- bq_table_download(bio_table, bigint = "integer64")
+
+bio_data$Connect_ID <- as.numeric(bio_data$Connect_ID)
+parts_data$Connect_ID <- as.numeric(parts_data$Connect_ID)
+
+
+
+#bioqc_join= left_join(parts_data, bio_data, by="Connect_ID") 
+
+#bioqc <- as_tibble(bioqc_join)
+
+bioqc= left_join(bio_data, parts_data,  by=c("Connect_ID", "d_827220437") )
+
+knitr::opts_chunk$set(comment = NA)
+
+
+
+
+
+bioqc <- bioqc %>%  mutate(Site=case_when(d_827220437==531629870 ~ "HealthPartners",
+                                                 d_827220437==548392715 ~ "Henry Ford Health System",
+                                                 d_827220437==303349821 ~ "Marshfield Clinical Health System",
+                                                 d_827220437==657167265 ~ "Sanford Health",
+                                                 d_827220437==809703864 ~ "University of Chicago Medicine",
+                                                d_827220437==125001209 ~ "Kaiser Permanente Colorado",
+                                                 d_827220437==327912200 ~ "Kaiser Permanente Georgia",
+                                                 d_827220437==300267574 ~ "Kaiser Permanente Hawaii",
+                                                 d_827220437==452412599 ~ "Kaiser Permanente Northwest"))
+
+```
+
+
+```{r, echo=FALSE, warning=FALSE, message=FALSE}
+
+
+#### IN A SEPARATE REPORT
+
+
+### HAWAII ONLY: They're sending one extra number
+
+#qc1 <- bioqc %>%  filter(d_827220437==300267574)
+
+#Blood
+qc1_blood <- bioqc %>%  filter(str_sub(d_173836415_d_266600170_d_341570479,1,11)!=str_sub(d_646899796_integer,1,11)) 
+
+qc1_blood <- qc1_blood %>% select(Site, Connect_ID, d_173836415_d_266600170_d_341570479, d_646899796_integer, d_556788178 ) %>%  sort_by(Site)
+
+
+# qc1_blood %>%  gt() %>%  
+#   tab_header(title = md("KPHI Blood Accession ID Errors")) %>% 
+#   cols_label(Connect_ID = md("**Connect ID**"), d_173836415_d_266600170_d_341570479 = md("**KPHI Accession ID**"), d_646899796_integer= md("**Expected Accession ID**"))
+
+colnames(qc1_blood) <- c("Site", "Connect ID", "KP Accession ID" , "Expected Accession ID", "Collection Date/Time")
+#knitr::kable(qc1_blood)
+
+write.csv(qc1_blood,paste(local_drive, "Mismatched_Blood_Accession_IDs",currentDate,".csv",sep="_"),row.names = F,na="")
+
+```
+
+
+
+```{r, echo=FALSE, warning=FALSE, message=FALSE}
+
+
+#### IN A SEPARATE REPORT
+
+
+
+
+
+#Urine
+qc1_urine <- bioqc %>%  filter(str_sub(d_173836415_d_266600170_d_198261154,1,11)!=str_sub(d_928693120_integer,1,11)) 
+
+qc1_urine$AID<- strtoi(qc1_urine$d_928693120_integer, base = 0L)
+
+qc1_urine <- qc1_urine %>% select(Site, Connect_ID, d_173836415_d_266600170_d_198261154, d_928693120_integer, d_556788178 ) %>%  sort_by(Site)
+
+# qc1_urine %>%  gt() %>%  
+#   tab_header(title = md("KPHI Urine Accession ID Errors")) %>% 
+#   cols_label(Connect_ID = md("**Connect ID**"), d_173836415_d_266600170_d_198261154 = md("**KPHI Accession ID**"), d_928693120_integer= md("**Expected Accession ID**"))
+
+colnames(qc1_urine) <- c("Site", "Connect ID", "KP Accession ID" , "Expected Accession ID", "Collection Date/Time")
+#knitr::kable(qc1_urine)
+
+write.csv(qc1_urine,paste(local_drive, "Mismatched_Urine_Accession_IDs",currentDate,".csv",sep="_"),row.names = F,na="")
+
+
+```
+
+
+
+
+
+
+
+```{r, include=FALSE}
+
+dup <- "WITH T AS (
+  SELECT Connect_ID FROM `nih-nci-dceg-connect-prod-6d04.FlatConnect.biospecimen_JP`
+  GROUP BY Connect_ID
+  HAVING COUNT(Connect_ID) > 1) 
+SELECT * FROM `nih-nci-dceg-connect-prod-6d04.FlatConnect.biospecimen_JP`
+INNER JOIN T ON `nih-nci-dceg-connect-prod-6d04.FlatConnect.biospecimen_JP`.Connect_ID = T.Connect_ID ;"
+
+
+dup_table <- bq_project_query(project, dup)
+dup_data <- bq_table_download(dup_table, bigint = "integer64")
+dup_data$Connect_ID <- as.numeric(dup_data$Connect_ID)
+
+
+bio_dup= left_join(dup_data, parts_data, by=c("Connect_ID", "d_827220437") )
+
+bio_dup <- bio_dup %>%  mutate(Site=case_when(d_827220437==531629870 ~ "HP",
+                                                 d_827220437==548392715 ~ "HF",
+                                                 d_827220437==303349821 ~ "Marshfield",
+                                                 d_827220437==657167265 ~ "Sanford",
+                                                 d_827220437==809703864 ~ "UChicago",
+                                                d_827220437==125001209 ~ "KPCO",
+                                                 d_827220437==327912200 ~ "KPGA",
+                                                 d_827220437==300267574 ~ "KPHI",
+                                                 d_827220437==452412599 ~ "KPNW"),
+                               Finalized= case_when(d_410912345==104430631 ~ "No",
+                                                    d_410912345==353358909 ~ "Yes",
+                                                    TRUE ~ "Not Enterted"))
+
+
+```
+
+
+```{r, echo=FALSE, warning=FALSE, message=FALSE}
+
+dup_list <- bio_dup %>%  select(Site, Connect_ID, d_820476880, d_556788178, Finalized)
+colnames(dup_list) <- c("Site", "Connect ID", "Collection ID", "Collection Date/Time", "Collection Finalized")
+#knitr::kable(dup_list)
+
+write.csv(dup_list,paste(local_drive, "Duplicate_Baseline_Biospecimen_Collections",currentDate,".csv",sep="_"),row.names = F,na="")
+
+```
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/get_merged_biospecimen_and_recruitment_data.R b/get_merged_biospecimen_and_recruitment_data.R
index f073130..a3f75a7 100644
--- a/get_merged_biospecimen_and_recruitment_data.R
+++ b/get_merged_biospecimen_and_recruitment_data.R
@@ -41,7 +41,8 @@ WITH
       d_173836415_d_266600170_d_448660695,
       d_684635302,
       d_878865966,
-      d_173836415_d_266600170_d_769615780
+      d_173836415_d_266600170_d_769615780,
+      d_827220437
     FROM `{project}.FlatConnect.participants_JP`
     WHERE Connect_ID IS NOT NULL
       AND d_831041022 = '104430631'
diff --git a/get_merged_blood_urine_data.R b/get_merged_blood_urine_data.R
new file mode 100644
index 0000000..295377b
--- /dev/null
+++ b/get_merged_blood_urine_data.R
@@ -0,0 +1,27 @@
+get_merged_blood_urine_data <-
+  function(project) {
+    billing <- project
+    
+    sql1 <- "
+            WITH
+                PART AS (
+                    SELECT Connect_ID, d_253883960, d_831041022, d_827220437, token
+                    FROM `{project}.FlatConnect.participants_JP`
+                    WHERE Connect_ID IS NOT NULL
+                      AND d_831041022 = '104430631'
+                      AND d_253883960='231311385'
+                ),
+                BU AS (
+                    SELECT *
+                    FROM `{project}.FlatConnect.clinicalBioSurvey_v1_JP`
+                    WHERE Connect_ID IS NOT NULL
+                )
+            SELECT *
+            FROM PART
+                LEFT JOIN BU ON PART.Connect_ID = BU.Connect_ID"
+    
+    data_query <- bq_project_query(project, query = glue(sql1))
+    data <- bq_table_download(data_query, bigint = "integer64")
+    
+    return(data)
+  }
\ No newline at end of file
diff --git a/get_merged_blood_urine_mouthwash_data.R b/get_merged_blood_urine_mouthwash_data.R
new file mode 100644
index 0000000..7d86889
--- /dev/null
+++ b/get_merged_blood_urine_mouthwash_data.R
@@ -0,0 +1,25 @@
+get_merged_blood_urine_mouthwash_data <-
+  function(project) {
+    billing <- project
+    
+    sql1 <- "
+            WITH
+                PART AS (
+                    SELECT Connect_ID, d_265193023, d_831041022, d_827220437, token
+                    FROM `{project}.FlatConnect.participants_JP`
+                    WHERE Connect_ID IS NOT NULL
+                      AND d_831041022 = '104430631'
+                      AND d_265193023='231311385'
+                ),
+                BUM AS (
+                    SELECT *
+                    FROM `{project}.FlatConnect.bioSurvey_v1_JP`
+                    WHERE Connect_ID IS NOT NULL
+                )
+            SELECT *
+            FROM PART
+                LEFT JOIN BUM ON PART.Connect_ID = BUM.Connect_ID"
+    
+    data_query <- bq_project_query(project, query = glue(sql1))
+    data <- bq_table_download(data_query, bigint = "integer64")
+  }
\ No newline at end of file
diff --git a/get_merged_covid_data.R b/get_merged_covid_data.R
new file mode 100644
index 0000000..1f91169
--- /dev/null
+++ b/get_merged_covid_data.R
@@ -0,0 +1,27 @@
+get_merged_covid_data <-
+  function(project) {
+    billing <- project
+    
+    sql1 <- "
+            WITH
+                PART AS (
+                    SELECT Connect_ID, d_220186468, d_831041022, d_827220437, token
+                    FROM `{project}.FlatConnect.participants_JP`
+                    WHERE Connect_ID IS NOT NULL
+                      AND d_831041022 = '104430631'
+                      AND d_220186468='231311385'
+                ),
+                CV AS (
+                    SELECT *
+                    FROM `{project}.FlatConnect.covid19Survey_v1_JP`
+                    WHERE Connect_ID IS NOT NULL
+                )
+            SELECT *
+            FROM PART
+                LEFT JOIN CV ON PART.Connect_ID = CV.Connect_ID"
+    
+    data_query <- bq_project_query(project, query = glue(sql1))
+    data <- bq_table_download(data_query, bigint = "integer64")
+    
+    return(data)
+  }
\ No newline at end of file
diff --git a/get_merged_menstrual_data.R b/get_merged_menstrual_data.R
new file mode 100644
index 0000000..6f5dd8f
--- /dev/null
+++ b/get_merged_menstrual_data.R
@@ -0,0 +1,27 @@
+get_merged_menstrual_data <-
+  function(project) {
+    billing <- project
+    
+    sql1 <- "
+            WITH
+                PART AS (
+                    SELECT Connect_ID, d_459098666, d_831041022, d_827220437, token
+                    FROM `{project}.FlatConnect.participants_JP`
+                    WHERE Connect_ID IS NOT NULL
+                      AND d_831041022 = '104430631'
+                      AND d_459098666='231311385'
+                ),
+                Mens AS (
+                    SELECT *
+                    FROM `{project}.FlatConnect.menstrualSurvey_v1_JP`
+                    WHERE Connect_ID IS NOT NULL
+                )
+            SELECT *
+            FROM PART
+                LEFT JOIN Mens ON PART.Connect_ID = Mens.Connect_ID"
+    
+    data_query <- bq_project_query(project, query = glue(sql1))
+    data <- bq_table_download(data_query, bigint = "integer64")
+    
+    return(data)
+  }
\ No newline at end of file
diff --git a/get_merged_module_2_data.R b/get_merged_module_2_data.R
index 27716e0..7664c2f 100644
--- a/get_merged_module_2_data.R
+++ b/get_merged_module_2_data.R
@@ -45,7 +45,7 @@ get_merged_module_2_data <- function(project) {
     query = glue(
       "SELECT Connect_ID, D_407056417, D_750420077_D_846483618,
               D_750420077_D_505282171, D_750420077_D_578416151, D_750420077_D_434651539,
-              D_750420077_D_108025529, D_289664241_D_289664241, D_613744428
+              D_750420077_D_108025529,  D_613744428
        FROM `{project}.FlatConnect.module1_v1_JP`
        WHERE Connect_ID IS NOT NULL")
   )
diff --git a/get_merged_module_3_data.R b/get_merged_module_3_data.R
index 827306f..f59d1ab 100644
--- a/get_merged_module_3_data.R
+++ b/get_merged_module_3_data.R
@@ -34,3 +34,4 @@ get_merged_module_3_data <- function(project) {
 # # Test
 # project <- "nih-nci-dceg-connect-prod-6d04"
 # df_mod3 <- get_merged_module_3_data(project)
+
diff --git a/get_merged_module_4_data.R b/get_merged_module_4_data.R
index 1b7c94f..38ae35a 100644
--- a/get_merged_module_4_data.R
+++ b/get_merged_module_4_data.R
@@ -21,7 +21,7 @@ get_merged_module_4_data <- function(project) {
     )
   parts_table <- bigrquery::bq_project_query(project, parts)
   parts_data <- bigrquery::bq_table_download(parts_table,
-                                  bigint = "integer64")
+                                             bigint = "integer64")
   
   mod4 <- glue::glue(
     "SELECT * FROM `{project}.FlatConnect.module4_v1_JP` ",
@@ -29,7 +29,7 @@ get_merged_module_4_data <- function(project) {
   )
   mod4_table <- bigrquery::bq_project_query(project, mod4)
   mod4_data <- bigrquery::bq_table_download(mod4_table,
-                                 bigint = "integer64")
+                                            bigint = "integer64")
   
   merged_mod4_parts_data <- dplyr::left_join(mod4_data, parts_data, by = "Connect_ID")
   
@@ -48,3 +48,4 @@ get_merged_module_4_data <- function(project) {
 # Test
 project <- "nih-nci-dceg-connect-prod-6d04"
 df_mod4 <- get_merged_module_4_data(project)
+
diff --git a/get_merged_mouthwash_data.R b/get_merged_mouthwash_data.R
new file mode 100644
index 0000000..3b2249d
--- /dev/null
+++ b/get_merged_mouthwash_data.R
@@ -0,0 +1,27 @@
+get_merged_mouthwash_data <-
+  function(project) {
+    billing <- project
+    
+    sql1 <- "
+            WITH
+                PART AS (
+                    SELECT Connect_ID, d_547363263, d_831041022, d_827220437, token
+                    FROM `{project}.FlatConnect.participants_JP`
+                    WHERE Connect_ID IS NOT NULL
+                      AND d_831041022 = '104430631'
+                      AND d_547363263='231311385'
+                ),
+                MW AS (
+                    SELECT *
+                    FROM `{project}.FlatConnect.mouthwash_v1_JP`
+                    WHERE Connect_ID IS NOT NULL
+                )
+            SELECT *
+            FROM PART
+                LEFT JOIN MW ON PART.Connect_ID = MW.Connect_ID"
+    
+    data_query <- bq_project_query(project, query = glue(sql1))
+    data <- bq_table_download(data_query, bigint = "integer64")
+    
+    return(data)
+  }
\ No newline at end of file
diff --git a/get_merged_rca_data.R b/get_merged_rca_data.R
index 030247f..b106a86 100644
--- a/get_merged_rca_data.R
+++ b/get_merged_rca_data.R
@@ -1,15 +1,16 @@
 get_merged_rca_data <- function(project){
   sql <- glue::glue(
-   "SELECT
+    "SELECT
       t.*,
-      p.d_827220437 -- get site_id
+      p.d_827220437, -- get site_id
+      p.token
     FROM `{project}.FlatConnect.cancerOccurrence_JP` AS t
     JOIN `{project}.FlatConnect.participants_JP` AS p
       ON t.Connect_ID = p.Connect_ID")
-
+  
   data_query <- bq_project_query(project, query = sql)
   data <- bq_table_download(data_query, bigint = "integer64")
-
+  
   print(glue("length data: {length(data)}"))
   return(data)
-}
+}
\ No newline at end of file
diff --git a/get_merged_screening_data.R b/get_merged_screening_data.R
new file mode 100644
index 0000000..ca9508f
--- /dev/null
+++ b/get_merged_screening_data.R
@@ -0,0 +1,27 @@
+get_merged_screening_data <-
+  function(project) {
+    billing <- project
+    
+    sql1 <- "
+            WITH
+                PART AS (
+                    SELECT Connect_ID, d_176068627, d_831041022, d_827220437, token
+                    FROM `{project}.FlatConnect.participants_JP`
+                    WHERE Connect_ID IS NOT NULL
+                      AND d_831041022 = '104430631'
+                      AND d_176068627='231311385'
+                ),
+                CSH AS (
+                    SELECT *
+                    FROM `{project}.FlatConnect.cancerScreeningHistorySurvey_JP`
+                    WHERE Connect_ID IS NOT NULL
+                )
+            SELECT *
+            FROM PART
+                LEFT JOIN CSH ON PART.Connect_ID = CSH.Connect_ID"
+    
+    data_query <- bq_project_query(project, query = glue(sql1))
+    data <- bq_table_download(data_query, bigint = "integer64")
+    
+    return(data)
+  }
\ No newline at end of file
diff --git a/prod/Dockerfile b/prod/Dockerfile
index 82773ba..d2e7e6d 100644
--- a/prod/Dockerfile
+++ b/prod/Dockerfile
@@ -9,8 +9,20 @@ COPY ["./get_merged_module_3_data.R", "./get_merged_module_3_data.R"]
 COPY ["./get_merged_module_4_data.R", "./get_merged_module_4_data.R"]
 COPY ["./get_merged_rca_data.R", "./get_merged_rca_data.R"]
 COPY ["./get_merged_biospecimen_and_recruitment_data.R", "./get_merged_biospecimen_and_recruitment_data.R"]
+COPY ["./get_merged_blood_urine_data.R", "./get_merged_blood_urine_data.R"]
+COPY ["./get_merged_blood_urine_mouthwash_data.R", "./get_merged_blood_urine_mouthwash_data.R"]
+COPY ["./get_merged_covid_data.R", "./get_merged_covid_data.R"]
+COPY ["./get_merged_menstrual_data.R", "./get_merged_menstrual_data.R"]
+COPY ["./get_merged_mouthwash_data.R", "./get_merged_mouthwash_data.R"]
+COPY ["./get_merged_screening_data.R", "./get_merged_screening_data.R"]
 COPY ["./check_and_correct_exceptions.R", "./check_and_correct_exceptions.R"]
 COPY ["./qc_rules_recruitment.xlsx", "./qc_rules_recruitment.xlsx"]
+COPY ["./qc_rules_blood_urine.xlsx", "./qc_rules_blood_urine.xlsx"]
+COPY ["./qc_rules_blood_urine_mouthwash.xlsx", "./qc_rules_blood_urine_mouthwash.xlsx"]
+COPY ["./qc_rules_covid_19.xlsx", "./qc_rules_covid_19.xlsx"]
+COPY ["./qc_rules_menstrual.xlsx", "./qc_rules_menstrual.xlsx"]
+COPY ["./qc_rules_mouthwash.xlsx", "./qc_rules_mouthwash.xlsx"]
+COPY ["./qc_rules_screening.xlsx", "./qc_rules_screening.xlsx"]
 COPY ["./qc_rules_biospecimen.xlsx", "./qc_rules_biospecimen.xlsx"]
 COPY ["./qc_rules_module1.xlsx", "./qc_rules_module1.xlsx"]
 COPY ["./qc_rules_module2.xlsx", "./qc_rules_module2.xlsx"]
diff --git a/prod/config.yml b/prod/config.yml
index dba0319..3e5fedc 100644
--- a/prod/config.yml
+++ b/prod/config.yml
@@ -29,6 +29,7 @@ module2:
 module3:
   QC_REPORT: module3
   rules_file: qc_rules_module3.xlsx
+  box_folder: "226704654836"
 module4:
   QC_REPORT: module4
   rules_file: qc_rules_module4.xlsx
@@ -36,6 +37,27 @@ module4:
 blood_urine_mouthwash:
   QC_REPORT: blood_urine_mouthwash
   rules_file: qc_rules_blood_urine_mouthwash.xlsx
+  box_folder: "270003455083"
+blood_urine:
+  QC_REPORT: blood_urine
+  rules_file: qc_rules_blood_urine.xlsx
+  box_folder: "270000834801"
+mouthwash:
+  QC_REPORT: mouthwash
+  rules_file: qc_rules_mouthwash.xlsx
+  box_folder: "270000582657"
+menstrual:
+  QC_REPORT: menstrual
+  rules_file: qc_rules_menstrual.xlsx
+  box_folder: "275062283610"
+covid:
+  QC_REPORT: covid
+  rules_file: qc_rules_covid_19.xlsx
+  box_folder: "270003995873"
+screening:
+  QC_REPORT: screening
+  rules_file: qc_rules_screening.xlsx
+  box_folder: "314365821322"
 rca:
   QC_REPORT: rca
   box_folder: "236420612811"
diff --git a/qaqc.R b/qaqc.R
index 78e3dd7..714bd09 100644
--- a/qaqc.R
+++ b/qaqc.R
@@ -1,12 +1,13 @@
 ################################################################################
 #############    Define Parameters Here IF Running Locally    ##################
 ################################################################################
-local_drive <- "/Users/petersjm/Documents/7_qaqc" #set to your working dir
+local_drive <- "C:/Users/dowlingk2/Documents/GitHub/qaqc_testing" #set to your working dir
 tier        <- "prod" # "prod" or "stg"
-module      <- "recruitment"
+module      <- "module4"
 # Options: "blood_urine_mouthwash", "biospecimen", "module1", "module2",
-#          "module3", "module4", "blood_urine_mouthwash", "rca"
-testing_api <- TRUE # ONLY SET TO TRUE IF YOU ARE TESTING PLUMBER API
+#          "module3", "module4", "blood_urine", "rca", 
+#           "mouthwash", "menstrual", "screening", "covid"
+testing_api <- FALSE # ONLY SET TO TRUE IF YOU ARE TESTING PLUMBER API
 ################################################################################
 ################################################################################
 
@@ -57,6 +58,7 @@ load_from_bq  <- config::get("LOAD_FROM_BQ")
 
 #Authenticate to bigrquery
 bq_auth()
+2
 
 # Name of output/report file
 rules_str  <- glue("rules{min_rule}to{max_rule}")
@@ -332,15 +334,14 @@ except_dates <- function(f) {
     if (!requireNamespace("lubridate", quietly = TRUE)) {
       stop("The 'lubridate' package is required but not installed.")
     }
-
+    
     l <- list(...)
     concept_col <- ifelse(!is.null(l$ConceptID), l$ConceptID, date_cid)
-
+    
     # Construct the comparison expression correctly
     expression_string <- glue::glue(
       "lubridate::ymd_hms({{ concept_col }}) {date_comparison} lubridate::ymd_hms('{date_to_compare}')"
     )
-
     f(data, ids = {{ ids }}, ...) %>%
       dplyr::filter(eval(parse(text = expression_string)))
   }
@@ -676,37 +677,37 @@ runQC <- function(data, rules, ids){
 loadData <- function(project, table, where_clause,
                      start_index = start_index,
                      n_max = n_max) {
-
-
-    if (n_max != Inf) {
-      LIMIT <- sprintf("LIMIT %s", as.character(as.integer(n_max)))
-    } else {
-      LIMIT <- ""
-    }
-
-    if (start_index > 0) {
-      OFFSET <- sprintf("OFFSET %s", as.character(as.integer(start_index)))
-    } else {
-      OFFSET <- ""
-    }
-
-    q <- glue("SELECT *
+  
+  
+  if (n_max != Inf) {
+    LIMIT <- sprintf("LIMIT %s", as.character(as.integer(n_max)))
+  } else {
+    LIMIT <- ""
+  }
+  
+  if (start_index > 0) {
+    OFFSET <- sprintf("OFFSET %s", as.character(as.integer(start_index)))
+  } else {
+    OFFSET <- ""
+  }
+  
+  q <- glue("SELECT *
                FROM `{project}.FlatConnect.{table}`
                {where_clause}
                -- ORDER BY token DESC
                -- {LIMIT}
                -- {OFFSET}
               ")
-    print(q)
-    tb   <- bq_project_query(project, query=q)
-    data <- bq_table_download(tb,
-                              bigint="integer64",
-                              page_size=1000,
-                              # start_index=start_index,
-                              n_max=n_max,
-                              quiet=TRUE)
-
-    return(data)
+  print(q)
+  tb   <- bq_project_query(project, query=q)
+  data <- bq_table_download(tb,
+                            bigint="integer64",
+                            page_size=1000,
+                            # start_index=start_index,
+                            n_max=n_max,
+                            quiet=TRUE)
+  
+  return(data)
 }
 
 get_explanation <- function(x, data) {
@@ -863,7 +864,6 @@ get_explanation <- function(x, data) {
 #### YOU WILL WANT TO CHANGE THIS TO TRUE...
 
 if (load_from_bq){
-
   if (QC_REPORT == "biospecimen") {
     source("get_merged_biospecimen_and_recruitment_data.R")
     data <- get_merged_biospecimen_and_recruitment_data(project,
@@ -878,7 +878,6 @@ if (load_from_bq){
                      start_index=start_index,
                      n_max=n_max)
   }
-
   else if (QC_REPORT == "rca") {
     print(glue("QC_REPORT = {QC_REPORT}"))
     source("get_merged_rca_data.R")
@@ -908,7 +907,6 @@ if (load_from_bq){
   else if (QC_REPORT == "module3") {
     source("get_merged_module_3_data.R")
     data <- get_merged_module_3_data(project)
-
     # Deal with exceptions
     source("check_and_correct_exceptions.R")
     exceptions <- config::get(value="exceptions", config = "module3")
@@ -918,7 +916,6 @@ if (load_from_bq){
   else if (QC_REPORT == "module4") {
     source("get_merged_module_4_data.R")
     data <- get_merged_module_4_data(project)
-
     # Deal with exceptions
     source("check_and_correct_exceptions.R")
     exceptions <- config::get(value="exceptions", config = "module4")
@@ -928,14 +925,60 @@ if (load_from_bq){
   else if (QC_REPORT == "blood_urine_mouthwash") {
     source("get_merged_blood_urine_mouthwash_data.R")
     data <- get_merged_blood_urine_mouthwash_data(project)
+    # Deal with exceptions
+    source("check_and_correct_exceptions.R")
     exceptions <- config::get(value="exceptions", config = "blood_urine_mouthwash")
     exceptions <- union(names(data), exceptions) # get only the ones that are actually in the df
     data <- check_and_correct_exceptions(data, exceptions)
   }
+  else if (QC_REPORT == "blood_urine") {
+    source("get_merged_blood_urine_data.R")
+    data <- get_merged_blood_urine_data(project)
+    # Deal with exceptions
+    source("check_and_correct_exceptions.R")
+    exceptions <- config::get(value="exceptions", config = "blood_urine")
+    exceptions <- union(names(data), exceptions) # get only the ones that are actually in the df
+    data <- check_and_correct_exceptions(data, exceptions)
+  }
+  else if (QC_REPORT == "mouthwash") {
+    source("get_merged_mouthwash_data.R")
+    data <- get_merged_mouthwash_data(project)
+    # Deal with exceptions
+    source("check_and_correct_exceptions.R")
+    exceptions <- config::get(value="exceptions", config = "mouthwash")
+    exceptions <- union(names(data), exceptions) # get only the ones that are actually in the df
+    data <- check_and_correct_exceptions(data, exceptions)
+  }
+  else if (QC_REPORT == "menstrual") {
+    source("get_merged_menstrual_data.R")
+    data <- get_merged_menstrual_data(project)
+    # Deal with exceptions
+    source("check_and_correct_exceptions.R")
+    exceptions <- config::get(value="exceptions", config = "menstrual")
+    exceptions <- union(names(data), exceptions) # get only the ones that are actually in the df
+    data <- check_and_correct_exceptions(data, exceptions)
+  }
+  else if (QC_REPORT == "covid") {
+    source("get_merged_covid_data.R")
+    data <- get_merged_covid_data(project)
+    # Deal with exceptions
+    source("check_and_correct_exceptions.R")
+    exceptions <- config::get(value="exceptions", config = "covid")
+    exceptions <- union(names(data), exceptions) # get only the ones that are actually in the df
+    data <- check_and_correct_exceptions(data, exceptions)
+  }
+  else if (QC_REPORT == "screening") {
+    source("get_merged_screening_data.R")
+    data <- get_merged_screening_data(project)
+    # Deal with exceptions
+    source("check_and_correct_exceptions.R")
+    exceptions <- config::get(value="exceptions", config = "screening")
+    exceptions <- union(names(data), exceptions) # get only the ones that are actually in the df
+    data <- check_and_correct_exceptions(data, exceptions)
+  }
   # Add a row of indices so that we can refer back to the original position later
   # after filtering
   # data$index <- 1:nrow(data)
-
 }
 
 ## I need to load the rules file....
@@ -993,8 +1036,6 @@ if (length(x)==0) {
     CrossVariableConceptValidValue4_lookup= map_chr(CrossVariableConceptValidValue4_lookup,paste,collapse = ", "),
   ) %>%
     get_explanation()
-
-
   # Alter column order
   col_order <- c("Connect_ID", "token", "rule_id",
                  "site_id", "site",
@@ -1006,7 +1047,6 @@ if (length(x)==0) {
                  "CrossVariableConceptID4", "CrossVariableConceptID4_lookup", "CrossVariableConceptValidValue4", "CrossVariableConceptValidValue4_lookup",
                  "date", "explanation")
   x <- x[, col_order]
-
   if (!is.null(exclusions_fid) && exclusions_fid != "") {
     # Read in exclusions file
     exclusions <- read.csv(exclusions_fid)
@@ -1041,4 +1081,4 @@ if (length(x)==0) {
     print("Successfully uploaded to GCS Bucket.")
   }
   rm(data)
-}
+}
\ No newline at end of file
diff --git a/qc_rules_blood_urine.xlsx b/qc_rules_blood_urine.xlsx
new file mode 100644
index 0000000..2c34c41
Binary files /dev/null and b/qc_rules_blood_urine.xlsx differ
diff --git a/qc_rules_blood_urine_mouthwash.xlsx b/qc_rules_blood_urine_mouthwash.xlsx
new file mode 100644
index 0000000..cdc3cef
Binary files /dev/null and b/qc_rules_blood_urine_mouthwash.xlsx differ
diff --git a/qc_rules_covid_19.xlsx b/qc_rules_covid_19.xlsx
new file mode 100644
index 0000000..edb0763
Binary files /dev/null and b/qc_rules_covid_19.xlsx differ
diff --git a/qc_rules_menstrual.xlsx b/qc_rules_menstrual.xlsx
new file mode 100644
index 0000000..ca68a22
Binary files /dev/null and b/qc_rules_menstrual.xlsx differ
diff --git a/qc_rules_module1.xlsx b/qc_rules_module1.xlsx
index 5b0efb0..855f526 100644
Binary files a/qc_rules_module1.xlsx and b/qc_rules_module1.xlsx differ
diff --git a/qc_rules_module2.xlsx b/qc_rules_module2.xlsx
index 9adcc96..17a82f4 100644
Binary files a/qc_rules_module2.xlsx and b/qc_rules_module2.xlsx differ
diff --git a/qc_rules_module3.xlsx b/qc_rules_module3.xlsx
index 8ecef79..69c92f5 100644
Binary files a/qc_rules_module3.xlsx and b/qc_rules_module3.xlsx differ
diff --git a/qc_rules_module4.xlsx b/qc_rules_module4.xlsx
index 05801b0..5414f8a 100644
Binary files a/qc_rules_module4.xlsx and b/qc_rules_module4.xlsx differ
diff --git a/qc_rules_mouthwash.xlsx b/qc_rules_mouthwash.xlsx
new file mode 100644
index 0000000..545afd9
Binary files /dev/null and b/qc_rules_mouthwash.xlsx differ
diff --git a/qc_rules_screening.xlsx b/qc_rules_screening.xlsx
new file mode 100644
index 0000000..37628cb
Binary files /dev/null and b/qc_rules_screening.xlsx differ


Reviews:
user: login: jacobmpeters
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-03T15:54:42Z
user: login: jacobmpeters
body: This looks good overall! Looks like you stuck to the formula for adding new modules and that consistency is important for maintainability. I asked a few questions in-line about the `loadData` function and the script that you included. 
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-03T15:56:45Z
user: login: KELSEYDOWLING7
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-03T15:58:23Z

Review comments:
diff_hunk: @@ -676,37 +677,37 @@ runQC <- function(data, rules, ids){
 loadData <- function(project, table, where_clause,
                      start_index = start_index,
                      n_max = n_max) {
-
-
-    if (n_max != Inf) {
-      LIMIT <- sprintf("LIMIT %s", as.character(as.integer(n_max)))
-    } else {
-      LIMIT <- ""
-    }
-
-    if (start_index > 0) {
-      OFFSET <- sprintf("OFFSET %s", as.character(as.integer(start_index)))
-    } else {
-      OFFSET <- ""
-    }
-
-    q <- glue("SELECT *
+  
+  
+  if (n_max != Inf) {
+    LIMIT <- sprintf("LIMIT %s", as.character(as.integer(n_max)))
+  } else {
+    LIMIT <- ""
+  }
+  
+  if (start_index > 0) {
+    OFFSET <- sprintf("OFFSET %s", as.character(as.integer(start_index)))
+  } else {
+    OFFSET <- ""
+  }
+  
+  q <- glue("SELECT *
                FROM `{project}.FlatConnect.{table}`
                {where_clause}
                -- ORDER BY token DESC
                -- {LIMIT}
                -- {OFFSET}
               ")
-    print(q)
-    tb   <- bq_project_query(project, query=q)
-    data <- bq_table_download(tb,
-                              bigint="integer64",
-                              page_size=1000,
-                              # start_index=start_index,
-                              n_max=n_max,
-                              quiet=TRUE)
-
-    return(data)
+  print(q)
+  tb   <- bq_project_query(project, query=q)
+  data <- bq_table_download(tb,
+                            bigint="integer64",
+                            page_size=1000,
+                            # start_index=start_index,
+                            n_max=n_max,
+                            quiet=TRUE)
+  
+  return(data)
 }
path: qaqc.R
user: login: jacobmpeters
body: You didn't make any changes here, right? This looks to just be small repositioning of the text.. this part is important for the recruitment QC, so I'd like to not change it if possible. 
created_at: 2025-04-03T15:54:42Z
author_association: CONTRIBUTOR
path: Biospecimen Metrics QC and Accession IDs.Rmd
user: login: jacobmpeters
body: Do you want this script in this repo?
created_at: 2025-04-03T15:55:22Z
author_association: CONTRIBUTOR
diff_hunk: @@ -676,37 +677,37 @@ runQC <- function(data, rules, ids){
 loadData <- function(project, table, where_clause,
                      start_index = start_index,
                      n_max = n_max) {
-
-
-    if (n_max != Inf) {
-      LIMIT <- sprintf("LIMIT %s", as.character(as.integer(n_max)))
-    } else {
-      LIMIT <- ""
-    }
-
-    if (start_index > 0) {
-      OFFSET <- sprintf("OFFSET %s", as.character(as.integer(start_index)))
-    } else {
-      OFFSET <- ""
-    }
-
-    q <- glue("SELECT *
+  
+  
+  if (n_max != Inf) {
+    LIMIT <- sprintf("LIMIT %s", as.character(as.integer(n_max)))
+  } else {
+    LIMIT <- ""
+  }
+  
+  if (start_index > 0) {
+    OFFSET <- sprintf("OFFSET %s", as.character(as.integer(start_index)))
+  } else {
+    OFFSET <- ""
+  }
+  
+  q <- glue("SELECT *
                FROM `{project}.FlatConnect.{table}`
                {where_clause}
                -- ORDER BY token DESC
                -- {LIMIT}
                -- {OFFSET}
               ")
-    print(q)
-    tb   <- bq_project_query(project, query=q)
-    data <- bq_table_download(tb,
-                              bigint="integer64",
-                              page_size=1000,
-                              # start_index=start_index,
-                              n_max=n_max,
-                              quiet=TRUE)
-
-    return(data)
+  print(q)
+  tb   <- bq_project_query(project, query=q)
+  data <- bq_table_download(tb,
+                            bigint="integer64",
+                            page_size=1000,
+                            # start_index=start_index,
+                            n_max=n_max,
+                            quiet=TRUE)
+  
+  return(data)
 }
path: qaqc.R
user: login: KELSEYDOWLING7
body: Correct, I didn't make any changes. I had only changed the input text options at the very top. So just adding the new reports in the comment at this part (lines 6-9):

(previously)
module      <- "recruitment"
# Options: "blood_urine_mouthwash", "biospecimen", "module1", "module2",
#          "module3", "module4", "blood_urine_mouthwash", "rca"

(now)
module      <- "module4"
# Options: "blood_urine_mouthwash", "biospecimen", "module1", "module2",
#          "module3", "module4", "blood_urine", "rca", 
#           "mouthwash", "menstrual", "screening", "covid"
created_at: 2025-04-03T15:58:23Z
author_association: CONTRIBUTOR

PR comments:
user: login: KELSEYDOWLING7
created_at: 2025-04-02T16:55:30Z
body: @jacobmpeters Can you please review this PR when you get a chance? I'd like to update at least the currently functioning automated qc for Monday. Thanks!
author_association: CONTRIBUTOR
user: login: jacobmpeters
created_at: 2025-04-02T16:57:16Z
body: > @jacobmpeters Can you please review this PR when you get a chance? I'd like to update at least the currently functioning automated qc for Monday. Thanks!

Yep! Thanks for the heads up. I'll take a look today.
author_association: CONTRIBUTOR