add txt of likely project IDs from portal pdfs
gave extension to pdf and txt
add pdf and txt filename to metadata.
these just have url encoding for forward slashes, and file extensions

while read line; do intermediate=$( echo "$line" | tawk '{print $4}' | sed 's:.*\(INPDFV\):\1:g' | sed 's:\/:%2F:g'); txt_name=$(echo "$intermediate".txt); pdf_name=$(echo "$intermediate".pdf);echo -e "$line\t$pdf_name\t$txt_name"; done < metadata.csv | sponge metadata.csv
make a TSV with name of txt file from pdf, text that is above and below a match, google translate of this text, and tagged versions of this text for likely DATE, PROJECT, and MONEY
NB: problem with translating text with '%', did not get to many texts this run

padding=5; find . -type f -iregex ".*"| parallel 'regex="Ministério das Relações Exteriores|Agência Brasileira de Cooperação|abc/mre|Cooperação Bloco|Cooperação Prestada Trilateral|Cooperação Sul-Sul|Prestada Trilateral com Organismo|BRA/[^ ]*"; txt=$( cat {} | tr "\r\n\f\t" " " ); padded=$( echo "$txt" | fmt -u -w 80 | grep -iE -A '$padding' -B '$padding' "$regex" | sed "s:^[ \t]\+::g;s:[ \t]\+$::g" ); txt_file_name=$(echo {} | sed "s:^[.]\/::g" ); function tag { tr "\n" " " | sed "s:\([A-Z]*\$\s*\+[0-9.,]*\):/MONEY/\1/MONEY/:g" | sed "s:\(BRA/[^ ]*\):/PROJECT/\1/PROJECT/:g" | sed "s:\s\([0-9]\{2\}\/[0-9]\{2\}\/[0-9]\{4\}\)\s: /DATE/\1/DATE/ :g" ;}; tagged_orig=$( echo "$padded" | tag ); trans=$( translate pt=en <(echo "$padded") | tr "\n" " "); tagged_trans=$( echo "$trans" | tag ); echo -e "$txt_file_name\t$(echo $padded | tr "\n" " ")\t$trans\t$tagged_orig\t$tagged_trans"' > ../../match.csv
Merge commit '0794077ee86d1fd67b64044a34f49ed173ebde14'