app.py

#$git clone https://huggingface.co/spaces/aboltachka/rr_detector 

#####################
# IMPORT PACKAGES
#####################

import gradio as gr
import pandas as pd
import re
import os
dirname = os.path.dirname(__file__)

#####################
# ALGORITHM SETUP
#####################

group_control = [('control',0,re.compile(r'\bminimum wage\b')),
           ('control',1,re.compile(r'\bcrime[a-zA-Z]{0,7}\b')),
           ('control',2,re.compile(r'\bimmigra[a-zA-Z]{0,4}\b')),
           ('control',3,re.compile(r'\bmigra[a-zA-Z]{0,4}\b')),
           ('control',4,re.compile(r'\bsingle mother[a-zA-Z]{0,1}\b')),
           ('control',5,re.compile(r'\blone mother[a-zA-Z]{0,1}\b')),
           ('control',6,re.compile(r'\bone parent[a-zA-Z]{0,3}\b')),
           ('control',7,re.compile(r'\blone parent[a-zA-Z]{0,3}\b')),
           ('control',8,re.compile(r'\bsingle parent[a-zA-Z]{0,3}\b')), #change from {0,1} to {0,3} for parenting
           ('control',9,re.compile(r'\bsingle headed\b')),
           ('control',10,re.compile(r'\blone headed\b')),
           ('control',11,re.compile(r'\bsingle headed\b')),
           ('control',12,re.compile(r'\blone headed\b')),
           ('control',13,re.compile(r'\bdoctor\b')),
           ('control',14,re.compile(r'\bphysician\b')),
           ('control',15,re.compile(r'\bself-employed\b')),
           ('control',16,re.compile(r'\bentrepreneur\b'))]

group_issue = [('issue',0,re.compile(r'\bdiscriminat[a-zA-Z]{0,5}\b')), # changed from {0,4} to {0,5}, e/g Discriminatively
           ('issue',1,re.compile(r'\bprejudi[a-zA-Z]{0,4}\b')), # changed from {0,3} to {0,4}, e/g Prejudicing
           ('issue',2,re.compile(r'\brac[a-zA-Z]{0,3} bias[a-zA-Z]{0,3}\b')),
           ('issue',3,re.compile(r'\brac[a-zA-Z]{0,3} disparit[a-zA-Z]{0,3}\b')),
           ('issue',4,re.compile(r'\brac[a-zA-Z]{0,3} stereotyp[a-zA-Z]{0,3}\b')),
           ('issue',5,re.compile(r'\breconstruction[a-zA-Z]{0,1}\b')),
           ('issue',6,re.compile(r'\bdesegregat[a-zA-Z]{0,3}\b')),
           ('issue',7,re.compile(r'\bjim crow\b')),
           ('issue',8,re.compile(r'\blynch[a-zA-Z]{0,5}\b')),   # changed from {0,3} to {0,5}, e/g lynchings
           ('issue',9,re.compile(r'\bwhitecapping\b')),
           ('issue',10,re.compile(r'\bcivil rights\b')),
           ('issue',11,re.compile(r'\brace riot[a-zA-Z]{0,3}\b')),
           ('issue',12,re.compile(r'\bemancipat[a-zA-Z]{0,3}\b')),
           ('issue',13,re.compile(r'\bslave[a-zA-Z]{0,2}\b')),
           ('issue',14,re.compile(r'\brace relation[a-zA-Z]{0,1}\b')),
           ('issue',15,re.compile(r'\bstatistical discrimination[a-zA-Z]{0,1}\b')),
           ('issue',16,re.compile(r'\banimus\b')),
           ('issue',17,re.compile(r'\banimosit[a-zA-Z]{0,3}\b')),
           ('issue',18,re.compile(r'\bsegregat[a-zA-Z]{0,3}\b')),
           ('issue',19,re.compile(r'\brac[a-zA-Z]{0,3} identit[a-zA-Z]{0,3}\b')),
           ('issue',21,re.compile(r'\beugenics\b')),
           ('issue',22,re.compile(r'\brac[a-zA-Z]{0,3} profiling\b')),
           ('issue',23,re.compile(r'\baffirmative action[a-zA-Z]{0,1}\b')),
           ('issue',24,re.compile(r'\btipping point\b')),
           ('issue',25,re.compile(r'\bblack-white\b')),
           ('issue',26,re.compile(r'\brac[a-zA-Z]{0,3} gap[a-zA-Z]{0,1}\b')),
           ('issue',27,re.compile(r'\brac[a-zA-Z]{0,3} differen[a-zA-Z]{0,4}\b')),
           ('issue',28,re.compile(r'\brac[a-zA-Z]{0,3} composition[a-zA-Z]{0,1}\b')),
           ('issue',29,re.compile(r'\brac[a-zA-Z]{0,3} integration[a-zA-Z]{0,1}\b')),
           ('issue',30,re.compile(r'\brac[a-zA-Z]{0,3} interact[a-zA-Z]{0,4}\b')),
           ('issue',31,re.compile(r'\brac[a-zA-Z]{0,3} inequalit[a-zA-Z]{0,3}\b')),
           ('issue',33,re.compile(r'\bnegro-white\b')),
           ('issue',34,re.compile(r'\banti-discrimination\b')),
           ('issue',35,re.compile(r'\bantidiscrimination\b')),
           ('issue',37,re.compile(r'\bethnic bias[a-zA-Z]{0,3}\b')),
           ('issue',38,re.compile(r'\bethnic disparit[a-zA-Z]{0,3}\b')),
           ('issue',39,re.compile(r'\bethnic stereotyp[a-zA-Z]{0,3}\b')),
           ('issue',40,re.compile(r'\bethnic gap[a-zA-Z]{0,1}\b')),
           ('issue',41,re.compile(r'\bethnic differen[a-zA-Z]{0,4}\b')),
           ('issue',42,re.compile(r'\bethnic composition[a-zA-Z]{0,1}\b')),
           ('issue',43,re.compile(r'\bethnic integration[a-zA-Z]{0,1}\b')),
           ('issue',44,re.compile(r'\bethnic interact[a-zA-Z]{0,4}\b')),
           ('issue',45,re.compile(r'\bethnic inequalit[a-zA-Z]{0,3}\b')),
           ('issue',46,re.compile(r'\bpostbellum\b')),
           ('issue',47,re.compile(r'\bdisadvantage\b')),
           ('issue',48,re.compile(r'\bdisadvantaged\b')),
           ('issue',49,re.compile(r'\battitude[a-zA-Z]{0,1}\b')),
           ('issue',50,re.compile(r'\bgap[a-zA-Z]{0,1}\b')),
           ('issue',51,re.compile(r'\bapartheid\b')),
           ('issue',52,re.compile(r'\brepresentation\b')),
           ('issue',53,re.compile(r'\bantisemitism\b')),
           ('issue',54,re.compile(r'\banti-semitic\b')),
           ('issue',55,re.compile(r'\banti-black\b')),
           ('issue',56,re.compile(r'\bidentity\b')),
           ('issue',57,re.compile(r'\bidentities\b')),
           ('issue',58,re.compile(r'\bsouthern farm\b')),
           ('issue',59,re.compile(r'\bracial heterogene[a-zA-Z]{0,5}\b')),
           ('issue',60,re.compile(r'\bethnic heterogene[a-zA-Z]{0,5}\b')),
           ('issue',61,re.compile(r'\bacting white\b')),
           ('issue',62,re.compile(r'\baffirmative-action[a-zA-Z]{0,1}\b')),
           ('issue',63,re.compile(r'\bhatred\b')),
           ('issue',64,re.compile(r'\bsocial activis[a-zA-Z]{0,1}\b')),
           ('issue',65,re.compile(r'\bsocial-activis[a-zA-Z]{0,1}\b')),
           ('issue',66,re.compile(r'\bethnic fragmentation[a-zA-Z]{0,1}\b')),
           ('issue',67,re.compile(r'\bethnic-fragmentation[a-zA-Z]{0,1}\b')),
           ('issue',68,re.compile(r'\bsocial fragmentation[a-zA-Z]{0,1}\b')),
           ('issue',69,re.compile(r'\bsocial-fragmentation[a-zA-Z]{0,1}\b')),
           ('issue',70,re.compile(r'\bracial fragmentation[a-zA-Z]{0,1}\b')),
           ('issue',71,re.compile(r'\bracial-fragmentation[a-zA-Z]{0,1}\b')),
           ('issue',72,re.compile(r'\bethnic division[a-zA-Z]{0,1}\b')),
           ('issue',73,re.compile(r'\bethnic-division[a-zA-Z]{0,1}\b')),
           ('issue',74,re.compile(r'\bsocial division[a-zA-Z]{0,1}\b')),
           ('issue',75,re.compile(r'\bsocial-division[a-zA-Z]{0,1}\b')),
           ('issue',76,re.compile(r'\bracial division[a-zA-Z]{0,1}\b')),
           ('issue',77,re.compile(r'\bracial-division[a-zA-Z]{0,1}\b')),
           ('issue',78,re.compile(r'\bethnic exclusion[a-zA-Z]{0,1}\b')),
           ('issue',79,re.compile(r'\bethnic-exclusion[a-zA-Z]{0,1}\b')),
           ('issue',80,re.compile(r'\bsocial exclusion[a-zA-Z]{0,1}\b')),
           ('issue',81,re.compile(r'\bsocial-exclusion[a-zA-Z]{0,1}\b')),
           ('issue',82,re.compile(r'\bracial exclusion[a-zA-Z]{0,1}\b')),
           ('issue',83,re.compile(r'\bracial-exclusion[a-zA-Z]{0,1}\b')),
           ('issue',84,re.compile(r'\bethnic diversity\b')),
           ('issue',85,re.compile(r'\bethnic-diversity\b')),
           ('issue',86,re.compile(r'\bsocial diversity\b')),
           ('issue',87,re.compile(r'\bsocial-diversity\b')),
           ('issue',88,re.compile(r'\bracial diversity\b')),
           ('issue',89,re.compile(r'\bracial-diversity\b')),
           ('issue',90,re.compile(r'\bthe great migration\b')),
           ('issue',91,re.compile(r'\bblack youth[a-zA-Z]{0,1}\b')),
           ('issue',92,re.compile(r'\bblack‐white\b')),
           ('issue',93,re.compile(r'\b-group bias\b')),
           ('issue',94,re.compile(r'\btuskegee\b')),
           ('issue',95,re.compile(r'\bingroup\b')),
           ('issue',96,re.compile(r'\bin-group\b')),
           ('issue',97,re.compile(r'\boutgroup\b')),
           ('issue',98,re.compile(r'\bout-group\b')),
           ('issue',99,re.compile(r'\binter-group\b')),
           ('issue',100,re.compile(r'\bintergroup\b')),
           ('issue',101,re.compile(r'\binequality\b')),
           ('issue',102,re.compile(r'\bstratification\b')),
           ('issue',103,re.compile(r'\bimplicit bias[a-zA-Z]{0,2}\b')),
           ('issue',104,re.compile(r'\bblack vot[a-zA-Z]{0,3}\b')),
           ('issue',105,re.compile(r'\bpolitical disenfranchisement\b')),
           ('issue',106,re.compile(r'\binstitutional discrimination\b')),
           ('issue',107,re.compile(r'\bstructural discrimination\b')),
           ('issue',108,re.compile(r'\binstitutional racism\b')),
           ('issue',109,re.compile(r'\bsystemic racism\b')),
           ('issue',110,re.compile(r'\bdifferential\b')),
           ('issue',111,re.compile(r'\b-differential\b')),
           ('issue',112,re.compile(r'\bunderrepresent[a-zA-Z]{0,3}\b')),
           ('issue',113,re.compile(r'\bexploitation\b')),
           ('issue',114,re.compile(r'\boppress[a-zA-Z]{0,3}\b')),
           ('issue',115, re.compile(r'\bschool[a-zA-Z]{0,3}\b')),
           ('issue',116, re.compile(r'\beducat[a-zA-Z]{0,5}\b')),
           ('issue',117, re.compile(r'\bdevelopment\b')),
           ('issue',118, re.compile(r'\bpoverty\b')),
           ('issue',119, re.compile(r'\bliving standard\b')),
           ('issue',120, re.compile(r'\bwelfare\b'))]

group_ethnicity = [('ethnicity',0,re.compile(r'\brac[a-zA-Z]{0,3}\b')),
           ('ethnicity',1,re.compile(r'\bafrican-american[a-zA-Z]{0,1}\b')),
           ('ethnicity',2,re.compile(r'\bafrican american[a-zA-Z]{0,1}\b')),
           ('ethnicity',3,re.compile(r'\bperson[a-zA-Z]{0,1} of color[a-zA-Z]{0,1}\b')),
           ('ethnicity',4,re.compile(r'\bethnic[a-zA-Z]{0,4}\b')),
           ('ethnicity',5,re.compile(r'\bhispanic[a-zA-Z]{0,1}\b')),
           ('ethnicity',6,re.compile(r'\blatino[a-zA-Z]{0,1}\b')),
           ('ethnicity',7,re.compile(r'\bblack[a-zA-Z]{0,1}\b')),
           #('ethnicity',8,re.compile(r'\bwhite[s]{0,1}\b')),
           ('ethnicity',11,re.compile(r'\bnegro[a-zA-Z]{0,2}\b')),
           ('ethnicity',15,re.compile(r'\bsouth asian[a-zA-Z]{0,1}\b')),
           ('ethnicity',17,re.compile(r'\bunderrepresented minorit[a-zA-Z]{0,3}\b')),
           ('ethnicity',18,re.compile(r'\bethnic minorit[a-zA-Z]{0,3}\b')),
           ('ethnicity',21,re.compile(r'\bblack-american[a-zA-Z]{0,1}\b')),
           ('ethnicity',22,re.compile(r'\bindian-american[a-zA-Z]{0,1}\b')),
           ('ethnicity',23,re.compile(r'\bjapanese-american[a-zA-Z]{0,1}\b')),
           ('ethnicity',24,re.compile(r'\bchinese-american[a-zA-Z]{0,1}\b')),
           ('ethnicity',25,re.compile(r'\bmexican-american[a-zA-Z]{0,1}\b')),
           ('ethnicity',26,re.compile(r'\bhispanic-american[a-zA-Z]{0,1}\b')),
           ('ethnicity',27,re.compile(r'\blatino-american[a-zA-Z]{0,1}\b')),
           ('ethnicity',28,re.compile(r'\bnative-american[a-zA-Z]{0,1}\b')),
           ('ethnicity',29,re.compile(r'\bamerican-indian[a-zA-Z]{0,1}\b')),
           ('ethnicity',30,re.compile(r'\borientals\b')),
           ('ethnicity',32,re.compile(r'\bkorean-american[a-zA-Z]{0,1}\b')),
           ('ethnicity',34,re.compile(r'\bvietnamese-american[a-zA-Z]{0,1}\b')),
           ('ethnicity',35,re.compile(r'\bnon-white[a-zA-Z]{0,1}\b')),
           ('ethnicity',36,re.compile(r'\bcolored[a-zA-Z]{0,1}\b')),
           ('ethnicity',37,re.compile(r'\bcaste[a-zA-Z]{0,1}\b')),
           ('ethnicity',38,re.compile(r'\bdisadvantaged minor[a-zA-Z]{0,5}\b')),
           ('ethnicity',39,re.compile(r'\badvantaged-group[a-zA-Z]{0,1}\b')),
           ('ethnicity',40,re.compile(r'\badvantaged group[a-zA-Z]{0,1}\b')),
           ('ethnicity',41,re.compile(r'\bdominant group[a-zA-Z]{0,1}\b')),
           ('ethnicity',42,re.compile(r'\bdominant-group[a-zA-Z]{0,1}\b')),
           ('ethnicity',43,re.compile(r'\bnon-western[a-zA-Z]{0,1}\b')),
           ('ethnicity',44,re.compile(r'\bblack american[a-zA-Z]{0,1}\b')),
           ('ethnicity',45,re.compile(r'\bindian american[a-zA-Z]{0,1}\b')),
           ('ethnicity',46,re.compile(r'\bjapanese american[a-zA-Z]{0,1}\b')),
           ('ethnicity',47,re.compile(r'\bchinese american[a-zA-Z]{0,1}\b')),
           ('ethnicity',48,re.compile(r'\bmexican american[a-zA-Z]{0,1}\b')),
           ('ethnicity',49,re.compile(r'\bhispanic american[a-zA-Z]{0,1}\b')),
           ('ethnicity',50,re.compile(r'\blatino american[a-zA-Z]{0,1}\b')),
           ('ethnicity',51,re.compile(r'\bnative american[a-zA-Z]{0,1}\b')),
           ('ethnicity',52,re.compile(r'\bamerican indian[a-zA-Z]{0,1}\b')),
           ('ethnicity',55,re.compile(r'\bnative[a-zA-Z]{0,1}\b')),
           ('ethnicity',56,re.compile(r'\bcaucasian[a-zA-Z]{0,1}\b')),
           ('ethnicity',57,re.compile(r'\bjew[a-zA-Z]{0,3}\b')),
           ('ethnicity',58,re.compile(r'\bhebrew[a-zA-Z]{0,1}\b')),
           ('ethnicity',59,re.compile(r'\byiddish\b')),
           ('ethnicity',60,re.compile(r'\bmuslim[a-zA-Z]{0,1}\b')),
           ('ethnicity',61,re.compile(r'\bislam[a-zA-Z]{0,2}\b')),
           ('ethnicity',62,re.compile(r'\bgay\b')),
           ('ethnicity',63,re.compile(r'\bqueer\b')),
           ('ethnicity',64,re.compile(r'\bhomosexual\b')),
           ('ethnicity',65,re.compile(r'\bhomo-sexual\b')),
           ('ethnicity',66,re.compile(r'\blesbian\b')),
           ('ethnicity',67,re.compile(r'\bpeople of colo[a-zA-Z]{0,1}r\b')),
           ('ethnicity',68,re.compile(r'\bpeople-of-colo[a-zA-Z]{0,1}r\b')),
           ('ethnicity',69,re.compile(r'\barab\b')),
           # aboriginal indigenous
           ('ethnicity',70,re.compile(r'\baboriginal\b')),
           ('ethnicity',71,re.compile(r'\bindigenous\b')),
           # tribes
           ('ethnicity',72,re.compile(r'\bnavajo[a-zA-Z]{0,3}\b')),
           ('ethnicity',73,re.compile(r'\bcherokee[a-zA-Z]{0,3}\b')),
           ('ethnicity',74,re.compile(r'\bcherokeean[a-zA-Z]{0,3}\b')),
           ('ethnicity',75,re.compile(r'\bsioux\b')),
           ('ethnicity',76,re.compile(r'\bsiouan\b')),
           ('ethnicity',77,re.compile(r'\bchippewa[a-zA-Z]{0,3}\b')),
           ('ethnicity',78,re.compile(r'\bchoctaw[a-zA-Z]{0,3}\b')),
           ('ethnicity',79,re.compile(r'\brace-related\b'))] #Added by Anton 

group_blackball = [('blackball',0, re.compile(r'\bblack.{0,3}market[a-zA-Z- ]{0,3}\b')),
                  ('blackball',1, re.compile(r'\bblack.{0,3}economy\b')),
                  ('blackball',2, re.compile(r'\bblack.{0,3}box.{0,3}\b')),
                  ('blackball',3, re.compile(r'\bblack.{0,3}card[a-zA-Z]{0,1}\b')),
                  ('blackball',4, re.compile(r'\bblack.{0,3}scholes\b')),
                  ('blackball',5, re.compile(r'\barms.{0,3}rac.{0,3}\b')),
                  ('blackball',6, re.compile(r'\bpatent.{0,3}rac.{0,3}\b')),
                  ('blackball',7, re.compile(r'\brat.{0,3}.{0,3}rac.{0,3}\b')),
                  ('blackball',8, re.compile(r'\bpriority.{0,3}rac.{0,3}\b')),
                  ('blackball',9, re.compile(r'\belectoral.{0,3}rac.{0,3}\b')),
                  ('blackball',10, re.compile(r'\brd.{0,3}rac.{0,3}\b')),
                  ('blackball',11, re.compile(r'\bwhite.{0,3}collar\b')),
                  ('blackball',12, re.compile(r'\bwhite.{0,3}noise\b')),
                  ('blackball',13, re.compile(r'\brace[s]{0,1} between\b')),
                  ('blackball',14, re.compile(r'\brac.*prize.{0,3}\b')),
                  ('blackball',15, re.compile(r'\bprize.*rac.{0,3}\b')),
                  ('blackball',16, re.compile(r'\brac.*winner{0,3}\b')),
                  ('blackball',17, re.compile(r'\bwinner.*rac.{0,3}\b')),
                  ('blackball',18, re.compile(r'\bhorse.*rac.{0,3}\b')),
                  ('blackball',19, re.compile(r'\brac.*horse.{0,3}\b')),
                  ('blackball',20, re.compile(r'\brival\b')),
                  ('blackball',21, re.compile(r'\br d.{0,3}rac.{0,3}\b')),
                  ('blackball',22, re.compile(r'\bblack swan\b'))]


search_regexs_groups = [group_control,group_issue,group_ethnicity,group_blackball]

group_main_american = [1,2,3,5,6,7,11,21,25,26,27,28,29,36,44,48,49,50,51,52,67,68,]
group_minor = [15,22,23,24,30,32,34,45,46,47,55,56,]
group_religious = [57,58,59,60,61,69,]
group_sexual = [62,63,64,65,66]
group_abstract = [0,4,17,18,35,36,37,38,39,40,41,42,43,70,71, 78]
group_tribal = [72,73,74,75,76,77,78]
group_0 = group_abstract
group_1 = group_abstract + group_main_american 
group_2 = group_abstract + group_main_american + group_tribal + group_minor + group_religious
group_3 = group_abstract + group_main_american + group_tribal + group_minor + group_religious + group_sexual
groups = [('0',group_0),('1',group_1),('2',group_2),('3',group_3),]
blackball_list = list(range(22))

#####################
# ALGORITHM FUNCTION
#####################

def rr_detector(title_raw, abstract_raw):
    title = title_raw.lower()
    abstract = abstract_raw.lower()
    result_dict = {}
    for g in search_regexs_groups:
        gname = g[0][0]
        any_ = 0
        any_ex_last = 0
        group_data = {}
        for group, code, search_regex in g:
            abstract_matches = search_regex.findall(abstract)
            abstract_count = len(abstract_matches)

            abstract_last_matches = search_regex.findall(abstract.split('. ')[-1])
            abstract_last_count = len(abstract_last_matches)

            title_matches = search_regex.findall(title)
            title_count = len(title_matches)

            any_ += abstract_count + title_count
            any_ex_last += abstract_count + title_count - abstract_last_count

            group_data['_'.join([group, str(code), 't', 'c'])] = title_count
            group_data['_'.join([group, str(code), 'at', 'c'])] = abstract_count + title_count
            group_data['_'.join([group, str(code), 'at_ex_last', 'c'])] = abstract_count + title_count - abstract_last_count
            group_data['_'.join([group, str(code), 'at', 'm'])] = int(abstract_count + title_count > 0)
            group_data['_'.join([group, str(code), 'at', 'match'])] = ';'.join(title_matches + abstract_matches)
        group_data['_'.join([gname, 's', 'at', 'm'])] = int(any_ > 0)
        group_data['_'.join([gname, 's', 'at', 'c'])] = any_
        group_data['_'.join([gname, 's', 'at_ex_last', 'm'])] = int(any_ex_last > 0)
        group_data['_'.join([gname, 's', 'at_ex_last', 'c'])] = any_ex_last

        result_dict.update(group_data)
        #result_dict[gname] = group_data

    results = {}
    results['blackballed'] = 0
    for gname, gvalue in result_dict.items():
        if 'blackball' in gname and '_at_c' in gname and 's' not in gname:
            if gvalue > 0 and results.get('blackballed', 0) != 1:
                results['blackballed'] = 1
    result_dict.update(results)

    results = {}
    for e, l in groups:
        results[f'group_{int(e)}'] = 0
        for i in l:
            if result_dict[f'ethnicity_{int(i)}_at_ex_last_c'] > 0 and results.get(f'group_{int(e)}', 0) != 1:
                results[f'group_{int(e)}'] = 1
    result_dict.update(results)

    results = {}
    for e,l in groups:
        results[f'match_{int(e)}'] = 0
        for i in l:
            if result_dict[f'ethnicity_{int(i)}_t_c'] > 0:
                if result_dict[f'group_{int(e)}']*result_dict['issue_s_at_m'] > 0 and results.get(f'match_{int(e)}', 0) != 1:
                    results[f'match_{int(e)}'] = 1
    result_dict.update(results)

    results = {}
    for e,l in groups:
        results[f'match_{int(e)}'] = (1-result_dict['blackballed'])*result_dict[f'match_{int(e)}']
    result_dict.update(results)

    #DETAILES ANALYSIS
    #Extract keywords (issue words)
    results_issue = []
    for gname, gvalue in result_dict.items():
        if 'issue_' in gname and 'at_match' in gname:
            results_issue.append(gvalue) 
    unique_issue = set()
    for item in results_issue:
        unique_issue.update(item.split(';'))
    issue_count = {} #<-- dictionary with unique issue keywords
    for term in unique_issue:
        count = 0
        for item in results_issue:
            count += item.count(term)
        issue_count[term] = count
    if '' in issue_count:
        del issue_count['']

    #Extract group (ethnic words)
    results_group = []
    for gname, gvalue in result_dict.items():
        if 'ethnicity_' in gname and 'at_match' in gname:
            results_group.append(gvalue) 
    unique_group = set()
    for item in results_group:
        unique_group.update(item.split(';'))
    group_count = {} #<-- dictionary with unique group keywords
    for term in unique_group:
        count = 0
        for item in results_group:
            count += item.count(term)
        group_count[term] = count
    if '' in group_count:
        del group_count[''] 

    #Extract blackball
    results_blackball = []
    for gname, gvalue in result_dict.items():
        if 'blackball_' in gname and 'at_match' in gname:
            results_blackball.append(gvalue) 
    unique_blackball = set()
    for item in results_blackball:
        unique_blackball.update(item.split(';'))
    blackball_count = {} #<-- dictionary with blackball keywords
    for term in unique_blackball:
        count = 0
        for item in results_blackball:
            count += item.count(term)
        blackball_count[term] = count
    if '' in blackball_count:
        del blackball_count[''] 

    #Append dictionaries into a data frame for Detailed statistics
    df_issue = pd.DataFrame(list(issue_count.items()), columns=['term', 'freq'])
    df_issue['type'] = 'issue'
    df_issue = df_issue[['type', 'term', 'freq']]

    df_group = pd.DataFrame(list(group_count.items()), columns=['term', 'freq'])
    df_group['type'] = 'group'
    df_group = df_group[['type', 'term', 'freq']]

    df_blackball = pd.DataFrame(list(blackball_count.items()), columns=['term', 'freq'])
    df_blackball['type'] = 'whitelist'
    df_blackball = df_blackball[['type', 'term', 'freq']]

    df_details = pd.concat([df_group, df_issue, df_blackball], ignore_index=True)

    #TEXT ANALYSIS
    #Dictionary with issue, topic, and blackball keywords
    keywords_dict = {"issue": [], "group": [], "whitelist": []}
    keywords_dict["issue"].extend(issue_count.keys())
    keywords_dict["group"].extend(group_count.keys())
    keywords_dict["whitelist"].extend(blackball_count.keys())

    combined_text = f"TITLE: {title_raw}. ABSTRACT: {abstract_raw}"

    text_analysis = []
    for word in combined_text.split():
        print(word)
        if word.lower() in [item.lower() for sublist in keywords_dict.values() for item in sublist]:
            for key, words in keywords_dict.items():
                if word.lower() in [item.lower() for item in words]:
                    text_analysis.append((word, key))
                    break
        else:
             text_analysis.append((word, None))
  
    #FORM THE MAIN OUTPUT
    #Output
    if result_dict['match_1'] == 1:
        #Result
        output_image = os.path.join(dirname, 'images/yes.png')
        #Explanation
        unique_group_str = ', '.join(unique_group)
        unique_issue_str = ', '.join(unique_issue)
        answer = "This paper can be considered race-related, as it mentions at least one group keyword AND one topic keywords in title or abstract. Furthermore, the algorithm does not identify any blackball phrases in the title and abstract provided."
        
    else:

        if len(blackball_count) > 0:
            #Result
            output_image = os.path.join(dirname, 'images/no.png')
            #Explanation
            unique_blackball_str = ', '.join(blackball_count)
            answer = "This paper cannot be considered race-related, as it includes the blackball phrase(s), such as: " + unique_blackball_str + "."
        else:
            #Result
            output_image = os.path.join(dirname, 'images/no.png')
            #Explanation
            answer = "This paper cannot be considered race-related, as it does not mention at least one group AND one topic keywords in title or abstract, or it does mention group keywords but only in the last sentence of provided abstract."
            #Details
            if len(issue_count.keys()) == 0 and len(group_count.keys()) == 0 and len(blackball_count.keys()) == 0 :
                data = {
                        "type": ["whitelist", "issue", "group"],
                        "term": ["term1", "term2", "term3"],
                        "freq": [0, 0, 0]
                        }
                df_details = pd.DataFrame(data)

        if len(abstract_raw) == 0:
            output_image = os.path.join(dirname, 'images/default.png')
            answer = "We need more information. Please submit abstact."
        if len(title_raw) == 0:
            output_image = os.path.join(dirname, 'images/default.png')
            answer = "We need more information. Please submit title."
        if len(title_raw) == 0 and len(abstract_raw) == 0:
            output_image = os.path.join(dirname, 'images/default.png')
            answer = "We need more information. Please submit title and abstract."

    return(output_image, answer, df_details, text_analysis)


#####################
# STYLE
#####################

def_image = os.path.join(dirname, 'images/default.png')


title_prompt = """
<!DOCTYPE html>
<html>
<head>
<style>
.title {
  font-family: Arial, sans-serif;
  font-size: 32px;
  font-weight: bold;
  text-align: center;
  letter-spacing: 3px;
  color: #333;
}
.subtitle {
  font-family: Arial, sans-serif;
  font-size: 18px;
  text-align: center;
  color: #666;
}
</style>
</head>
<body>

<div class="title">IDENTIFYING RACE-RELATED RESEARCH</div>

</body>
</html>
"""

description_prompt = """
<p>This app is supplementary material to the <strong>"Race-related Research in Economics" paper</strong>, where we examine how academic economists contribute to discussions about racial justice and enduring economic disparities among different racial and ethnic groups. Specifically, we analyze the production of race-related research in Economics. Our study is based on the analysis of a corpus of 250,000 economics publications from 1960 to 2020, employing an algorithmic approach to classify race-related publications. <strong>This app enables users to verify whether their research can be categorized as race-related based on our algorithm</strong>.</p>

<p>If you would like our algorithm to classify your research, please submit the title and abstract of your paper. By default, the title and abstract of our paper are provided, and you can verify whether it is a race-related research.</p>

"""



#####################
# APP LAUNCH
#####################

title_smpl = "Race-related Research in Economics"
abstract_smpl = "Issues of racial justice and persistent economic inequalities across racial and ethnic groups have risen to the top of public debate. The ability of academic economists to contribute to these debates in part depends on the production of race-related research in the profession. We study the issue combining information on a corpus of 250,000 publications in economics from 1960 to 2020 on which we use an algorithmic approach to classify race-related publications, constructing paths to publication for 22,000 NBER working papers between 1974 and 2015, and constructing the career prole of publications of 2800 economics faculty in US economics departments active in 2020/1. We present four new stylized facts on race-related research in economics. First, since 1960 less than 2% of publications in economics have been race related, with an uptick in such work since the mid 1990s. This represents a cumulative body of knowledge of 3801 race-related publications in economics since 1960. Second, the publications process provides little disincentive to produce race-related research: such work has similar or better publication outcomes as non race-related research. Third, Black faculty are significantly more likely to publish race-related work during their career. However, citations and H-indices are significantly lower for minority faculty as a whole. However, the citation penalty for Black faculty is partially offset for their race-related publications. Fourth, over later stages of the career life cycle, Black faculty become less likely to work on race-related topics. The timing of this change coincides with their career progression up the ranking of US academic departments. We draw together policy implications for the profession related to innovative areas of race-related research that economists can engage in, and processes to improve the selection and retention of minority faculty."
 

demo = gr.Interface(fn=rr_detector, inputs=[
        gr.Textbox(label="Title", value=title_smpl, lines=2),
        gr.Textbox(label="Abstract", value=abstract_smpl, lines=18)], 
    outputs=[
        gr.Image(label = 'Result', value=def_image),
        gr.Textbox(label="Explanation"),
        gr.BarPlot(
            label = "Details",
            x="term",
            y="freq",
            color="type",
            title="Frequency of Race-related Keywords in Title and Abstract",
            tooltip=["type", "term", "freq"],
            vertical=False,
            #caption = "TEST",
            height = 150,
            width = 300,
            color_legend_title = 'Type of Keywords',
            x_title = "Keywords",
            y_title = "Frequency"
        ),
        gr.HighlightedText(
        label="Text Analysis",
        show_legend=True,
        color_map={"group": "yellow", "issue": "blue", "whitelist": "grey"}),
    ], theme='Jameswiller/Globe', title = title_prompt, description = description_prompt, allow_flagging = 'auto')

#theme='gradio/monochrome'
#theme='ParityError/Interstellar'

if __name__ == "__main__":
    demo.launch(share=True)

'''
# Add default picture for output 
# Output as graph of just text but with fancy representation -- use labels from theme
# Generate picts for output with GenAi

#RR
title_raw = 'When expectations work race and socioeconomic differences in school performance'
abstract_raw = 'Why race between are expectations for future performance realized more often by some people than by others and why are such differences in the efficacy of performance expectations socially patterned we hypothesize that differences in attentiveness to performance feedback may be relevant reasoning that follow-through behaviors will be less well conceived when expectations are formed without regard to evaluation of previous performance. using data from baltimore fourth-grade students and their parents we find that expectations anticipate marks more accurately when recall of prior marks is correct than when it is incorrect. because errors of recall mostly on the high side are more common among lower-ses and minority children and their parents their school performance is affected most strongly. research on school attainment process from a motivational perspective must give more attention to the additional resources that facilitate successful goal attainment given high expectations. our perspective focuses on resources internal to the individual but external constraints also are important. the discussion stresses the need for further work in both areas.'

title_raw = "Race-related Research in Economics disadvantaged minor race disparity"
abstract_raw = "Issues of race disparity "


#Default
title_raw = "Race-related Research in Economics"
abstract_raw = "Issues of racial justice and persistent economic inequalities across racial and ethnic groups have risen to the top of public debate. The ability of academic economists to contribute to these debates in part depends on the production of race-related research in the profession. We study the issue combining information on a corpus of 250,000 publications in economics from 1960 to 2020 on which we use an algorithmic approach to classify race-related publications, constructing paths to publication for 22,000 NBER working papers between 1974 and 2015, and constructing the career prole of publications of 2800 economics faculty in US economics departments active in 2020/1. We present four new stylized facts on race-related research in economics."



#non-RR
title_raw = 'Hurting stalemate or mediation the conflict over nagorno-karabakh 1990-95'
abstract_raw = 'The impacts of six attempts to mediate the conflict over the political status of nagorno-karabakh in the caucasus region of the former soviet union were compared. each mediation was intended to get the direct parties armenia azerbaijan and nagorno-karabakh to the negotiating table. nearly 4000 events were recorded for a six-year period from 1990 through 1995. each event was coded in terms of a six-step scale ranging from a significant action toward peace 3 to substantial violence directed at an adversary -3. time-series analyses of changes in the extent of violence showed no change from before to after any of the mediations. a significant change did occur however between the months preceding and following the period of intensive combat between april 1993 and february 1994. these results support the hypothesis that a mutually hurting stalemate is a condition for negotiating a ceasefire and reduced violence between warring parties. a number of theoretical and practical implications of the findings are discussed.'

title_raw = ""
abstract_raw = ""

rr_detector(title_raw, abstract_raw)

'''





#TEXT ANALYSIS -- IMPROVE

# Graph: looks like when it is two words, it double count it: (this paper is about racial inequality, this paper is about racial inequality)
    #PROBLEM OF DOUBLE COUNT: GROUP (disadvantaged minor[a-zA-Z]{0,5}) and ISSUE (disadvantage)



def highlight_words(sentence, words):
    for i in range(len(sentence)):
        for j in range(len(words)):    
            if sentence.lower().startswith(words[j].lower(), i):
                sentence = sentence[:i] + sentence[i:i+len(words[j])].upper() + sentence[i+len(words[j]):]
    return sentence

print(highlight_words("Have a nIcE day, you Nice person!!", ["nice"]))
print(highlight_words("Shhh, don't be so loud!", ["loud", "Be"]))
print(highlight_words("Automating with Python is fun", ["fun", "auTomaTiNG"]))