Spaces:
Sleeping
Sleeping
| #$git clone https://huggingface.co/spaces/aboltachka/rr_detector | |
| ##################### | |
| # IMPORT PACKAGES | |
| ##################### | |
| import gradio as gr | |
| import pandas as pd | |
| import re | |
| import os | |
| dirname = os.path.dirname(__file__) | |
| ##################### | |
| # ALGORITHM SETUP | |
| ##################### | |
| group_control = [('control',0,re.compile(r'\bminimum wage\b')), | |
| ('control',1,re.compile(r'\bcrime[a-zA-Z]{0,7}\b')), | |
| ('control',2,re.compile(r'\bimmigra[a-zA-Z]{0,4}\b')), | |
| ('control',3,re.compile(r'\bmigra[a-zA-Z]{0,4}\b')), | |
| ('control',4,re.compile(r'\bsingle mother[a-zA-Z]{0,1}\b')), | |
| ('control',5,re.compile(r'\blone mother[a-zA-Z]{0,1}\b')), | |
| ('control',6,re.compile(r'\bone parent[a-zA-Z]{0,3}\b')), | |
| ('control',7,re.compile(r'\blone parent[a-zA-Z]{0,3}\b')), | |
| ('control',8,re.compile(r'\bsingle parent[a-zA-Z]{0,3}\b')), #change from {0,1} to {0,3} for parenting | |
| ('control',9,re.compile(r'\bsingle headed\b')), | |
| ('control',10,re.compile(r'\blone headed\b')), | |
| ('control',11,re.compile(r'\bsingle headed\b')), | |
| ('control',12,re.compile(r'\blone headed\b')), | |
| ('control',13,re.compile(r'\bdoctor\b')), | |
| ('control',14,re.compile(r'\bphysician\b')), | |
| ('control',15,re.compile(r'\bself-employed\b')), | |
| ('control',16,re.compile(r'\bentrepreneur\b'))] | |
| group_issue = [('issue',0,re.compile(r'\bdiscriminat[a-zA-Z]{0,5}\b')), # changed from {0,4} to {0,5}, e/g Discriminatively | |
| ('issue',1,re.compile(r'\bprejudi[a-zA-Z]{0,4}\b')), # changed from {0,3} to {0,4}, e/g Prejudicing | |
| ('issue',2,re.compile(r'\brac[a-zA-Z]{0,3} bias[a-zA-Z]{0,3}\b')), | |
| ('issue',3,re.compile(r'\brac[a-zA-Z]{0,3} disparit[a-zA-Z]{0,3}\b')), | |
| ('issue',4,re.compile(r'\brac[a-zA-Z]{0,3} stereotyp[a-zA-Z]{0,3}\b')), | |
| ('issue',5,re.compile(r'\breconstruction[a-zA-Z]{0,1}\b')), | |
| ('issue',6,re.compile(r'\bdesegregat[a-zA-Z]{0,3}\b')), | |
| ('issue',7,re.compile(r'\bjim crow\b')), | |
| ('issue',8,re.compile(r'\blynch[a-zA-Z]{0,5}\b')), # changed from {0,3} to {0,5}, e/g lynchings | |
| ('issue',9,re.compile(r'\bwhitecapping\b')), | |
| ('issue',10,re.compile(r'\bcivil rights\b')), | |
| ('issue',11,re.compile(r'\brace riot[a-zA-Z]{0,3}\b')), | |
| ('issue',12,re.compile(r'\bemancipat[a-zA-Z]{0,3}\b')), | |
| ('issue',13,re.compile(r'\bslave[a-zA-Z]{0,2}\b')), | |
| ('issue',14,re.compile(r'\brace relation[a-zA-Z]{0,1}\b')), | |
| ('issue',15,re.compile(r'\bstatistical discrimination[a-zA-Z]{0,1}\b')), | |
| ('issue',16,re.compile(r'\banimus\b')), | |
| ('issue',17,re.compile(r'\banimosit[a-zA-Z]{0,3}\b')), | |
| ('issue',18,re.compile(r'\bsegregat[a-zA-Z]{0,3}\b')), | |
| ('issue',19,re.compile(r'\brac[a-zA-Z]{0,3} identit[a-zA-Z]{0,3}\b')), | |
| ('issue',21,re.compile(r'\beugenics\b')), | |
| ('issue',22,re.compile(r'\brac[a-zA-Z]{0,3} profiling\b')), | |
| ('issue',23,re.compile(r'\baffirmative action[a-zA-Z]{0,1}\b')), | |
| ('issue',24,re.compile(r'\btipping point\b')), | |
| ('issue',25,re.compile(r'\bblack-white\b')), | |
| ('issue',26,re.compile(r'\brac[a-zA-Z]{0,3} gap[a-zA-Z]{0,1}\b')), | |
| ('issue',27,re.compile(r'\brac[a-zA-Z]{0,3} differen[a-zA-Z]{0,4}\b')), | |
| ('issue',28,re.compile(r'\brac[a-zA-Z]{0,3} composition[a-zA-Z]{0,1}\b')), | |
| ('issue',29,re.compile(r'\brac[a-zA-Z]{0,3} integration[a-zA-Z]{0,1}\b')), | |
| ('issue',30,re.compile(r'\brac[a-zA-Z]{0,3} interact[a-zA-Z]{0,4}\b')), | |
| ('issue',31,re.compile(r'\brac[a-zA-Z]{0,3} inequalit[a-zA-Z]{0,3}\b')), | |
| ('issue',33,re.compile(r'\bnegro-white\b')), | |
| ('issue',34,re.compile(r'\banti-discrimination\b')), | |
| ('issue',35,re.compile(r'\bantidiscrimination\b')), | |
| ('issue',37,re.compile(r'\bethnic bias[a-zA-Z]{0,3}\b')), | |
| ('issue',38,re.compile(r'\bethnic disparit[a-zA-Z]{0,3}\b')), | |
| ('issue',39,re.compile(r'\bethnic stereotyp[a-zA-Z]{0,3}\b')), | |
| ('issue',40,re.compile(r'\bethnic gap[a-zA-Z]{0,1}\b')), | |
| ('issue',41,re.compile(r'\bethnic differen[a-zA-Z]{0,4}\b')), | |
| ('issue',42,re.compile(r'\bethnic composition[a-zA-Z]{0,1}\b')), | |
| ('issue',43,re.compile(r'\bethnic integration[a-zA-Z]{0,1}\b')), | |
| ('issue',44,re.compile(r'\bethnic interact[a-zA-Z]{0,4}\b')), | |
| ('issue',45,re.compile(r'\bethnic inequalit[a-zA-Z]{0,3}\b')), | |
| ('issue',46,re.compile(r'\bpostbellum\b')), | |
| ('issue',47,re.compile(r'\bdisadvantage\b')), | |
| ('issue',48,re.compile(r'\bdisadvantaged\b')), | |
| ('issue',49,re.compile(r'\battitude[a-zA-Z]{0,1}\b')), | |
| ('issue',50,re.compile(r'\bgap[a-zA-Z]{0,1}\b')), | |
| ('issue',51,re.compile(r'\bapartheid\b')), | |
| ('issue',52,re.compile(r'\brepresentation\b')), | |
| ('issue',53,re.compile(r'\bantisemitism\b')), | |
| ('issue',54,re.compile(r'\banti-semitic\b')), | |
| ('issue',55,re.compile(r'\banti-black\b')), | |
| ('issue',56,re.compile(r'\bidentity\b')), | |
| ('issue',57,re.compile(r'\bidentities\b')), | |
| ('issue',58,re.compile(r'\bsouthern farm\b')), | |
| ('issue',59,re.compile(r'\bracial heterogene[a-zA-Z]{0,5}\b')), | |
| ('issue',60,re.compile(r'\bethnic heterogene[a-zA-Z]{0,5}\b')), | |
| ('issue',61,re.compile(r'\bacting white\b')), | |
| ('issue',62,re.compile(r'\baffirmative-action[a-zA-Z]{0,1}\b')), | |
| ('issue',63,re.compile(r'\bhatred\b')), | |
| ('issue',64,re.compile(r'\bsocial activis[a-zA-Z]{0,1}\b')), | |
| ('issue',65,re.compile(r'\bsocial-activis[a-zA-Z]{0,1}\b')), | |
| ('issue',66,re.compile(r'\bethnic fragmentation[a-zA-Z]{0,1}\b')), | |
| ('issue',67,re.compile(r'\bethnic-fragmentation[a-zA-Z]{0,1}\b')), | |
| ('issue',68,re.compile(r'\bsocial fragmentation[a-zA-Z]{0,1}\b')), | |
| ('issue',69,re.compile(r'\bsocial-fragmentation[a-zA-Z]{0,1}\b')), | |
| ('issue',70,re.compile(r'\bracial fragmentation[a-zA-Z]{0,1}\b')), | |
| ('issue',71,re.compile(r'\bracial-fragmentation[a-zA-Z]{0,1}\b')), | |
| ('issue',72,re.compile(r'\bethnic division[a-zA-Z]{0,1}\b')), | |
| ('issue',73,re.compile(r'\bethnic-division[a-zA-Z]{0,1}\b')), | |
| ('issue',74,re.compile(r'\bsocial division[a-zA-Z]{0,1}\b')), | |
| ('issue',75,re.compile(r'\bsocial-division[a-zA-Z]{0,1}\b')), | |
| ('issue',76,re.compile(r'\bracial division[a-zA-Z]{0,1}\b')), | |
| ('issue',77,re.compile(r'\bracial-division[a-zA-Z]{0,1}\b')), | |
| ('issue',78,re.compile(r'\bethnic exclusion[a-zA-Z]{0,1}\b')), | |
| ('issue',79,re.compile(r'\bethnic-exclusion[a-zA-Z]{0,1}\b')), | |
| ('issue',80,re.compile(r'\bsocial exclusion[a-zA-Z]{0,1}\b')), | |
| ('issue',81,re.compile(r'\bsocial-exclusion[a-zA-Z]{0,1}\b')), | |
| ('issue',82,re.compile(r'\bracial exclusion[a-zA-Z]{0,1}\b')), | |
| ('issue',83,re.compile(r'\bracial-exclusion[a-zA-Z]{0,1}\b')), | |
| ('issue',84,re.compile(r'\bethnic diversity\b')), | |
| ('issue',85,re.compile(r'\bethnic-diversity\b')), | |
| ('issue',86,re.compile(r'\bsocial diversity\b')), | |
| ('issue',87,re.compile(r'\bsocial-diversity\b')), | |
| ('issue',88,re.compile(r'\bracial diversity\b')), | |
| ('issue',89,re.compile(r'\bracial-diversity\b')), | |
| ('issue',90,re.compile(r'\bthe great migration\b')), | |
| ('issue',91,re.compile(r'\bblack youth[a-zA-Z]{0,1}\b')), | |
| ('issue',92,re.compile(r'\bblack‐white\b')), | |
| ('issue',93,re.compile(r'\b-group bias\b')), | |
| ('issue',94,re.compile(r'\btuskegee\b')), | |
| ('issue',95,re.compile(r'\bingroup\b')), | |
| ('issue',96,re.compile(r'\bin-group\b')), | |
| ('issue',97,re.compile(r'\boutgroup\b')), | |
| ('issue',98,re.compile(r'\bout-group\b')), | |
| ('issue',99,re.compile(r'\binter-group\b')), | |
| ('issue',100,re.compile(r'\bintergroup\b')), | |
| ('issue',101,re.compile(r'\binequality\b')), | |
| ('issue',102,re.compile(r'\bstratification\b')), | |
| ('issue',103,re.compile(r'\bimplicit bias[a-zA-Z]{0,2}\b')), | |
| ('issue',104,re.compile(r'\bblack vot[a-zA-Z]{0,3}\b')), | |
| ('issue',105,re.compile(r'\bpolitical disenfranchisement\b')), | |
| ('issue',106,re.compile(r'\binstitutional discrimination\b')), | |
| ('issue',107,re.compile(r'\bstructural discrimination\b')), | |
| ('issue',108,re.compile(r'\binstitutional racism\b')), | |
| ('issue',109,re.compile(r'\bsystemic racism\b')), | |
| ('issue',110,re.compile(r'\bdifferential\b')), | |
| ('issue',111,re.compile(r'\b-differential\b')), | |
| ('issue',112,re.compile(r'\bunderrepresent[a-zA-Z]{0,3}\b')), | |
| ('issue',113,re.compile(r'\bexploitation\b')), | |
| ('issue',114,re.compile(r'\boppress[a-zA-Z]{0,3}\b')), | |
| ('issue',115, re.compile(r'\bschool[a-zA-Z]{0,3}\b')), | |
| ('issue',116, re.compile(r'\beducat[a-zA-Z]{0,5}\b')), | |
| ('issue',117, re.compile(r'\bdevelopment\b')), | |
| ('issue',118, re.compile(r'\bpoverty\b')), | |
| ('issue',119, re.compile(r'\bliving standard\b')), | |
| ('issue',120, re.compile(r'\bwelfare\b'))] | |
| group_ethnicity = [('ethnicity',0,re.compile(r'\brac[a-zA-Z]{0,3}\b')), | |
| ('ethnicity',1,re.compile(r'\bafrican-american[a-zA-Z]{0,1}\b')), | |
| ('ethnicity',2,re.compile(r'\bafrican american[a-zA-Z]{0,1}\b')), | |
| ('ethnicity',3,re.compile(r'\bperson[a-zA-Z]{0,1} of color[a-zA-Z]{0,1}\b')), | |
| ('ethnicity',4,re.compile(r'\bethnic[a-zA-Z]{0,4}\b')), | |
| ('ethnicity',5,re.compile(r'\bhispanic[a-zA-Z]{0,1}\b')), | |
| ('ethnicity',6,re.compile(r'\blatino[a-zA-Z]{0,1}\b')), | |
| ('ethnicity',7,re.compile(r'\bblack[a-zA-Z]{0,1}\b')), | |
| #('ethnicity',8,re.compile(r'\bwhite[s]{0,1}\b')), | |
| ('ethnicity',11,re.compile(r'\bnegro[a-zA-Z]{0,2}\b')), | |
| ('ethnicity',15,re.compile(r'\bsouth asian[a-zA-Z]{0,1}\b')), | |
| ('ethnicity',17,re.compile(r'\bunderrepresented minorit[a-zA-Z]{0,3}\b')), | |
| ('ethnicity',18,re.compile(r'\bethnic minorit[a-zA-Z]{0,3}\b')), | |
| ('ethnicity',21,re.compile(r'\bblack-american[a-zA-Z]{0,1}\b')), | |
| ('ethnicity',22,re.compile(r'\bindian-american[a-zA-Z]{0,1}\b')), | |
| ('ethnicity',23,re.compile(r'\bjapanese-american[a-zA-Z]{0,1}\b')), | |
| ('ethnicity',24,re.compile(r'\bchinese-american[a-zA-Z]{0,1}\b')), | |
| ('ethnicity',25,re.compile(r'\bmexican-american[a-zA-Z]{0,1}\b')), | |
| ('ethnicity',26,re.compile(r'\bhispanic-american[a-zA-Z]{0,1}\b')), | |
| ('ethnicity',27,re.compile(r'\blatino-american[a-zA-Z]{0,1}\b')), | |
| ('ethnicity',28,re.compile(r'\bnative-american[a-zA-Z]{0,1}\b')), | |
| ('ethnicity',29,re.compile(r'\bamerican-indian[a-zA-Z]{0,1}\b')), | |
| ('ethnicity',30,re.compile(r'\borientals\b')), | |
| ('ethnicity',32,re.compile(r'\bkorean-american[a-zA-Z]{0,1}\b')), | |
| ('ethnicity',34,re.compile(r'\bvietnamese-american[a-zA-Z]{0,1}\b')), | |
| ('ethnicity',35,re.compile(r'\bnon-white[a-zA-Z]{0,1}\b')), | |
| ('ethnicity',36,re.compile(r'\bcolored[a-zA-Z]{0,1}\b')), | |
| ('ethnicity',37,re.compile(r'\bcaste[a-zA-Z]{0,1}\b')), | |
| ('ethnicity',38,re.compile(r'\bdisadvantaged minor[a-zA-Z]{0,5}\b')), | |
| ('ethnicity',39,re.compile(r'\badvantaged-group[a-zA-Z]{0,1}\b')), | |
| ('ethnicity',40,re.compile(r'\badvantaged group[a-zA-Z]{0,1}\b')), | |
| ('ethnicity',41,re.compile(r'\bdominant group[a-zA-Z]{0,1}\b')), | |
| ('ethnicity',42,re.compile(r'\bdominant-group[a-zA-Z]{0,1}\b')), | |
| ('ethnicity',43,re.compile(r'\bnon-western[a-zA-Z]{0,1}\b')), | |
| ('ethnicity',44,re.compile(r'\bblack american[a-zA-Z]{0,1}\b')), | |
| ('ethnicity',45,re.compile(r'\bindian american[a-zA-Z]{0,1}\b')), | |
| ('ethnicity',46,re.compile(r'\bjapanese american[a-zA-Z]{0,1}\b')), | |
| ('ethnicity',47,re.compile(r'\bchinese american[a-zA-Z]{0,1}\b')), | |
| ('ethnicity',48,re.compile(r'\bmexican american[a-zA-Z]{0,1}\b')), | |
| ('ethnicity',49,re.compile(r'\bhispanic american[a-zA-Z]{0,1}\b')), | |
| ('ethnicity',50,re.compile(r'\blatino american[a-zA-Z]{0,1}\b')), | |
| ('ethnicity',51,re.compile(r'\bnative american[a-zA-Z]{0,1}\b')), | |
| ('ethnicity',52,re.compile(r'\bamerican indian[a-zA-Z]{0,1}\b')), | |
| ('ethnicity',55,re.compile(r'\bnative[a-zA-Z]{0,1}\b')), | |
| ('ethnicity',56,re.compile(r'\bcaucasian[a-zA-Z]{0,1}\b')), | |
| ('ethnicity',57,re.compile(r'\bjew[a-zA-Z]{0,3}\b')), | |
| ('ethnicity',58,re.compile(r'\bhebrew[a-zA-Z]{0,1}\b')), | |
| ('ethnicity',59,re.compile(r'\byiddish\b')), | |
| ('ethnicity',60,re.compile(r'\bmuslim[a-zA-Z]{0,1}\b')), | |
| ('ethnicity',61,re.compile(r'\bislam[a-zA-Z]{0,2}\b')), | |
| ('ethnicity',62,re.compile(r'\bgay\b')), | |
| ('ethnicity',63,re.compile(r'\bqueer\b')), | |
| ('ethnicity',64,re.compile(r'\bhomosexual\b')), | |
| ('ethnicity',65,re.compile(r'\bhomo-sexual\b')), | |
| ('ethnicity',66,re.compile(r'\blesbian\b')), | |
| ('ethnicity',67,re.compile(r'\bpeople of colo[a-zA-Z]{0,1}r\b')), | |
| ('ethnicity',68,re.compile(r'\bpeople-of-colo[a-zA-Z]{0,1}r\b')), | |
| ('ethnicity',69,re.compile(r'\barab\b')), | |
| # aboriginal indigenous | |
| ('ethnicity',70,re.compile(r'\baboriginal\b')), | |
| ('ethnicity',71,re.compile(r'\bindigenous\b')), | |
| # tribes | |
| ('ethnicity',72,re.compile(r'\bnavajo[a-zA-Z]{0,3}\b')), | |
| ('ethnicity',73,re.compile(r'\bcherokee[a-zA-Z]{0,3}\b')), | |
| ('ethnicity',74,re.compile(r'\bcherokeean[a-zA-Z]{0,3}\b')), | |
| ('ethnicity',75,re.compile(r'\bsioux\b')), | |
| ('ethnicity',76,re.compile(r'\bsiouan\b')), | |
| ('ethnicity',77,re.compile(r'\bchippewa[a-zA-Z]{0,3}\b')), | |
| ('ethnicity',78,re.compile(r'\bchoctaw[a-zA-Z]{0,3}\b'))] | |
| group_blackball = [('blackball',0, re.compile(r'\bblack.{0,3}market[a-zA-Z- ]{0,3}\b')), | |
| ('blackball',1, re.compile(r'\bblack.{0,3}economy\b')), | |
| ('blackball',2, re.compile(r'\bblack.{0,3}box.{0,3}\b')), | |
| ('blackball',3, re.compile(r'\bblack.{0,3}card[a-zA-Z]{0,1}\b')), | |
| ('blackball',4, re.compile(r'\bblack.{0,3}scholes\b')), | |
| ('blackball',5, re.compile(r'\barms.{0,3}rac.{0,3}\b')), | |
| ('blackball',6, re.compile(r'\bpatent.{0,3}rac.{0,3}\b')), | |
| ('blackball',7, re.compile(r'\brat.{0,3}.{0,3}rac.{0,3}\b')), | |
| ('blackball',8, re.compile(r'\bpriority.{0,3}rac.{0,3}\b')), | |
| ('blackball',9, re.compile(r'\belectoral.{0,3}rac.{0,3}\b')), | |
| ('blackball',10, re.compile(r'\brd.{0,3}rac.{0,3}\b')), | |
| ('blackball',11, re.compile(r'\bwhite.{0,3}collar\b')), | |
| ('blackball',12, re.compile(r'\bwhite.{0,3}noise\b')), | |
| ('blackball',13, re.compile(r'\brace[s]{0,1} between\b')), | |
| ('blackball',14, re.compile(r'\brac.*prize.{0,3}\b')), | |
| ('blackball',15, re.compile(r'\bprize.*rac.{0,3}\b')), | |
| ('blackball',16, re.compile(r'\brac.*winner{0,3}\b')), | |
| ('blackball',17, re.compile(r'\bwinner.*rac.{0,3}\b')), | |
| ('blackball',18, re.compile(r'\bhorse.*rac.{0,3}\b')), | |
| ('blackball',19, re.compile(r'\brac.*horse.{0,3}\b')), | |
| ('blackball',20, re.compile(r'\brival\b')), | |
| ('blackball',21, re.compile(r'\br d.{0,3}rac.{0,3}\b')), | |
| ('blackball',22, re.compile(r'\bblack swan\b'))] | |
| search_regexs_groups = [group_control,group_issue,group_ethnicity,group_blackball] | |
| group_main_american = [1,2,3,5,6,7,11,21,25,26,27,28,29,36,44,48,49,50,51,52,67,68,] | |
| group_minor = [15,22,23,24,30,32,34,45,46,47,55,56,] | |
| group_religious = [57,58,59,60,61,69,] | |
| group_sexual = [62,63,64,65,66] | |
| group_abstract = [0,4,17,18,35,36,37,38,39,40,41,42,43,70,71, 78] | |
| group_tribal = [72,73,74,75,76,77,78] | |
| group_0 = group_abstract | |
| group_1 = group_abstract + group_main_american | |
| group_2 = group_abstract + group_main_american + group_tribal + group_minor + group_religious | |
| group_3 = group_abstract + group_main_american + group_tribal + group_minor + group_religious + group_sexual | |
| groups = [('0',group_0),('1',group_1),('2',group_2),('3',group_3),] | |
| blackball_list = list(range(22)) | |
| ##################### | |
| # ALGORITHM FUNCTION | |
| ##################### | |
| def rr_detector(title_raw, abstract_raw): | |
| title = title_raw.lower() | |
| abstract = abstract_raw.lower() | |
| result_dict = {} | |
| for g in search_regexs_groups: | |
| gname = g[0][0] | |
| any_ = 0 | |
| any_ex_last = 0 | |
| group_data = {} | |
| for group, code, search_regex in g: | |
| abstract_matches = search_regex.findall(abstract) | |
| abstract_count = len(abstract_matches) | |
| abstract_last_matches = search_regex.findall(abstract.split('. ')[-1]) | |
| abstract_last_count = len(abstract_last_matches) | |
| title_matches = search_regex.findall(title) | |
| title_count = len(title_matches) | |
| any_ += abstract_count + title_count | |
| any_ex_last += abstract_count + title_count - abstract_last_count | |
| group_data['_'.join([group, str(code), 't', 'c'])] = title_count | |
| group_data['_'.join([group, str(code), 'at', 'c'])] = abstract_count + title_count | |
| group_data['_'.join([group, str(code), 'at_ex_last', 'c'])] = abstract_count + title_count - abstract_last_count | |
| group_data['_'.join([group, str(code), 'at', 'm'])] = int(abstract_count + title_count > 0) | |
| group_data['_'.join([group, str(code), 'at', 'match'])] = ';'.join(title_matches + abstract_matches) | |
| group_data['_'.join([gname, 's', 'at', 'm'])] = int(any_ > 0) | |
| group_data['_'.join([gname, 's', 'at', 'c'])] = any_ | |
| group_data['_'.join([gname, 's', 'at_ex_last', 'm'])] = int(any_ex_last > 0) | |
| group_data['_'.join([gname, 's', 'at_ex_last', 'c'])] = any_ex_last | |
| result_dict.update(group_data) | |
| #result_dict[gname] = group_data | |
| results = {} | |
| results['blackballed'] = 0 | |
| for gname, gvalue in result_dict.items(): | |
| if 'blackball' in gname and '_at_c' in gname and 's' not in gname: | |
| if gvalue > 0 and results.get('blackballed', 0) != 1: | |
| results['blackballed'] = 1 | |
| result_dict.update(results) | |
| results = {} | |
| for e, l in groups: | |
| results[f'group_{int(e)}'] = 0 | |
| for i in l: | |
| if result_dict[f'ethnicity_{int(i)}_at_ex_last_c'] > 0 and results.get(f'group_{int(e)}', 0) != 1: | |
| results[f'group_{int(e)}'] = 1 | |
| result_dict.update(results) | |
| results = {} | |
| for e,l in groups: | |
| results[f'match_{int(e)}'] = 0 | |
| for i in l: | |
| if result_dict[f'ethnicity_{int(i)}_t_c'] > 0 and results.get(f'match_{int(e)}', 0) != 1 : | |
| results[f'match_{int(e)}'] = 1 | |
| tag = 'case1' #at least one GROUP keyword is in the title | |
| elif result_dict[f'group_{int(e)}']*result_dict['issue_s_at_m'] > 0 and results.get(f'match_{int(e)}', 0) != 1: | |
| results[f'match_{int(e)}'] = 1 | |
| tag = 'case2' #OR at least one group keyword and at least one issue keyword are mentioned in the title or abstract | |
| result_dict.update(results) | |
| results = {} | |
| for e,l in groups: | |
| results[f'match_{int(e)}'] = (1-result_dict['blackballed'])*result_dict[f'match_{int(e)}'] | |
| result_dict.update(results) | |
| #DETAILES ANALYSIS | |
| #Extract keywords (issue words) | |
| results_issue = [] | |
| for gname, gvalue in result_dict.items(): | |
| if 'issue_' in gname and 'at_match' in gname: | |
| results_issue.append(gvalue) | |
| unique_issue = set() | |
| for item in results_issue: | |
| unique_issue.update(item.split(';')) | |
| issue_count = {} #<-- dictionary with unique issue keywords | |
| for term in unique_issue: | |
| count = 0 | |
| for item in results_issue: | |
| count += item.count(term) | |
| issue_count[term] = count | |
| if '' in issue_count: | |
| del issue_count[''] | |
| #Extract group (ethnic words) | |
| results_group = [] | |
| for gname, gvalue in result_dict.items(): | |
| if 'ethnicity_' in gname and 'at_match' in gname: | |
| results_group.append(gvalue) | |
| unique_group = set() | |
| for item in results_group: | |
| unique_group.update(item.split(';')) | |
| group_count = {} #<-- dictionary with unique group keywords | |
| for term in unique_group: | |
| count = 0 | |
| for item in results_group: | |
| count += item.count(term) | |
| group_count[term] = count | |
| if '' in group_count: | |
| del group_count[''] | |
| #Extract blackball | |
| results_blackball = [] | |
| for gname, gvalue in result_dict.items(): | |
| if 'blackball_' in gname and 'at_match' in gname: | |
| results_blackball.append(gvalue) | |
| unique_blackball = set() | |
| for item in results_blackball: | |
| unique_blackball.update(item.split(';')) | |
| blackball_count = {} #<-- dictionary with blackball keywords | |
| for term in unique_blackball: | |
| count = 0 | |
| for item in results_blackball: | |
| count += item.count(term) | |
| blackball_count[term] = count | |
| if '' in blackball_count: | |
| del blackball_count[''] | |
| #Append dictionaries into a data frame for Detailed statistics | |
| df_issue = pd.DataFrame(list(issue_count.items()), columns=['term', 'freq']) | |
| df_issue['type'] = 'ISSUE' | |
| df_issue = df_issue[['type', 'term', 'freq']] | |
| df_group = pd.DataFrame(list(group_count.items()), columns=['term', 'freq']) | |
| df_group['type'] = 'GROUP' | |
| df_group = df_group[['type', 'term', 'freq']] | |
| df_blackball = pd.DataFrame(list(blackball_count.items()), columns=['term', 'freq']) | |
| df_blackball['type'] = 'EXCEPTION' | |
| df_blackball = df_blackball[['type', 'term', 'freq']] | |
| df_details = pd.concat([df_group, df_issue, df_blackball], ignore_index=True) | |
| issue_default = {'type': 'ISSUE', 'term': '', 'freq': ''} | |
| group_default = {'type': 'GROUP', 'term': '', 'freq': ''} | |
| blackball_default = {'type': 'EXCEPTION', 'term': '', 'freq': ''} | |
| df_details.loc[len(df_details)] = issue_default | |
| df_details.loc[len(df_details)] = group_default | |
| df_details.loc[len(df_details)] = blackball_default | |
| df_details = df_details.sort_values(by='type', ascending=True) | |
| #TEXT ANALYSIS | |
| #Dictionary with issue, topic, and blackball keywords | |
| keywords_dict = {"issue": [], "group": [], "exception": []} | |
| keywords_dict["issue"].extend(issue_count.keys()) | |
| keywords_dict["group"].extend(group_count.keys()) | |
| keywords_dict["exception"].extend(blackball_count.keys()) | |
| combined_text = f"TITLE:\n{title_raw} \n \nABSTRACT:\n{abstract_raw}" | |
| keywords = [(word, key, len(word)) for key, words in keywords_dict.items() for word in words] | |
| keywords = sorted(keywords, key=lambda x: -x[2]) | |
| if len(keywords) > 0: | |
| pattern = re.compile("|".join(map(re.escape, [x[0] for x in keywords])), re.IGNORECASE) | |
| matches = re.finditer(pattern, combined_text) | |
| text_analysis = [] | |
| last_end = 0 | |
| for match in matches: | |
| start = match.start() | |
| end = match.end() | |
| if start != last_end: | |
| text_analysis.append((combined_text[last_end:start], None)) | |
| for keyword, key, length in keywords: | |
| if re.match(re.escape(keyword), match.group(), re.IGNORECASE): | |
| text_analysis.append((combined_text[start:end], key)) | |
| break | |
| last_end = end | |
| if last_end != len(combined_text): | |
| text_analysis.append((combined_text[last_end:], None)) | |
| else: | |
| text_analysis = [(combined_text, None)] | |
| #FORM THE MAIN OUTPUT | |
| #Output | |
| if result_dict['match_1'] == 1: | |
| #Result | |
| output_image = os.path.join(dirname, 'images/yes.png') | |
| #Explanation | |
| unique_group_str = ', '.join(unique_group) | |
| unique_issue_str = ', '.join(unique_issue) | |
| answer = "This paper can be considered race-related, as it mentions at least one group keyword in the title. Or it mentions at least one group keyword AND at least one issue keyword in the title or abstract (excluding the last sentence). Furthermore, the algorithm does not identify any exception phrases in the title and abstract provided." | |
| else: | |
| if len(blackball_count) > 0: | |
| #Result | |
| output_image = os.path.join(dirname, 'images/no.png') | |
| #Explanation | |
| unique_blackball_str = ', '.join(blackball_count) | |
| answer = "This paper cannot be considered race-related, as it includes the exception phrase(s), such as: " + unique_blackball_str + "." | |
| else: | |
| #Result | |
| output_image = os.path.join(dirname, 'images/no.png') | |
| #Explanation | |
| answer = "This paper cannot be considered race-related, as it does not mention at least one group in the title, or it does not mention at least one group AND one topic keywords in title or abstract (excluding the last sentence), or it does mention group keywords but only in the last sentence of provided abstract." | |
| #Details | |
| if len(issue_count.keys()) == 0 and len(group_count.keys()) == 0 and len(blackball_count.keys()) == 0 : | |
| data = { | |
| "type": ["EXCEPTION", "ISSUE", "GROUP"], | |
| "term": ["term1", "term2", "term3"], | |
| "freq": [0, 0, 0] | |
| } | |
| df_details = pd.DataFrame(data) | |
| if len(abstract_raw) == 0: | |
| output_image = os.path.join(dirname, 'images/default.png') | |
| answer = "We need more information. Please submit abstact." | |
| if len(title_raw) == 0: | |
| output_image = os.path.join(dirname, 'images/default.png') | |
| answer = "We need more information. Please submit title." | |
| if len(title_raw) == 0 and len(abstract_raw) == 0: | |
| output_image = os.path.join(dirname, 'images/default.png') | |
| answer = "We need more information. Please submit title and abstract." | |
| return(output_image, answer, df_details, text_analysis) | |
| ##################### | |
| # STYLE | |
| ##################### | |
| def_image = os.path.join(dirname, 'images/default.png') | |
| title_prompt = """ | |
| <!DOCTYPE html> | |
| <html> | |
| <head> | |
| <style> | |
| .title { | |
| font-family: Arial, sans-serif; | |
| font-size: 32px; | |
| font-weight: bold; | |
| text-align: center; | |
| letter-spacing: 3px; | |
| color: #333; | |
| } | |
| .subtitle { | |
| font-family: Arial, sans-serif; | |
| font-size: 18px; | |
| text-align: center; | |
| color: #666; | |
| } | |
| </style> | |
| </head> | |
| <body> | |
| <div class="title">IDENTIFYING RACE-RELATED RESEARCH</div> | |
| </body> | |
| </html> | |
| """ | |
| description_prompt = """ | |
| <p>This app is supplementary material to the <strong>"Race-related Research in Economics" paper</strong>, where we examine how academic economists contribute to discussions about racial justice and enduring economic disparities among different racial and ethnic groups. Specifically, we analyze the production of race-related research in Economics. Our study is based on the analysis of a corpus of 250,000 economics publications from 1960 to 2020, employing an algorithmic approach to classify race-related publications. <strong>This app enables users to verify whether their research can be categorized as race-related based on our algorithm</strong>.</p> | |
| <p>If you would like our algorithm to classify your research, please submit the title and abstract of your paper. By default, the title and abstract of Bertrand and Mullainathan (2004) are provided, and you can verify whether it is a race-related research.</p> | |
| """ | |
| ##################### | |
| # APP LAUNCH | |
| ##################### | |
| title_smpl = "Are Emily and Greg More Employable Than Lakisha and Jamal? A Field Experiment on Labor Market Discrimination" | |
| abstract_smpl = "We study race in the labor market by sending fictitious resumes to help-wanted ads in Boston and Chicago newspapers. To manipulate perceived race, resumes are randomly assigned African-American- or White-sounding names. White names receive 50 percent more callbacks for interviews. Callbacks are also more responsive to resume quality for White names than for African-American ones. The racial gap is uniform across occupation, industry, and employer size. We also find little evidence that employers are inferring social class from the names. Differential treatment by race still appears to still be prominent in the U. S. labor market." | |
| demo = gr.Interface(fn=rr_detector, inputs=[ | |
| gr.Textbox(label="Title", value=title_smpl, lines=1), | |
| gr.Textbox(label="Abstract", value=abstract_smpl, lines=18)], | |
| outputs=[ | |
| gr.Image(label = 'Result', value=def_image), | |
| gr.Textbox(label="Explanation"), | |
| gr.BarPlot( | |
| label = "Details", | |
| x="term", | |
| y="freq", | |
| color="type", | |
| title="Frequency of Race-related Keywords in Title and Abstract", | |
| tooltip=["type", "term", "freq"], | |
| vertical=False, | |
| #caption = "TEST", | |
| height = 150, | |
| width = 300, | |
| color_legend_title = 'Type of Keywords', | |
| x_title = "Keywords", | |
| y_title = "Frequency", | |
| show_label = True, | |
| #sort = '-x', | |
| color_legend_position = 'right', | |
| ), | |
| gr.HighlightedText( | |
| label="Text Analysis", | |
| color_map = {'group': 'blue', 'issue': 'green', 'exception': 'red'} | |
| ), | |
| ], theme='Jameswiller/Globe', title = title_prompt, description = description_prompt, allow_flagging = 'auto') | |
| #theme='gradio/monochrome' | |
| #theme='ParityError/Interstellar' | |
| if __name__ == "__main__": | |
| demo.launch(share=True) | |
| ''' | |
| 1. Double count when two word keywords | |
| Title: Race inequality as a concept | |
| Abstract: This paper is about race inequality. | |
| ''' |