use only np projid where have content from url
rename according to detox
convert HTML and PDF to plain text
$ find . -type f|parallel 'type=$(file {} | grep -oE "[^:]*$" | awk "{print \$1}" | sed "s:^[ \t]\+::g" ); if [[ "$type" == "PDF" ]]; then pdftotext {}; rm {}; elif [[ "$type" == "HTML" ]]; then html2text {} | sponge {}; fi'
make a single txt file with projid for text of html and pdf that were available from source url fields for np
$ ls | parallel 'txt=$( cat $( find {} -type f ) | tr "\n" " " | sed "s:\t: :g;s:$:\n:g" ); echo -e {}"\t$txt"'|tawk '{if($2 !~ /^$/)print $0}' > /tmp/np_txt.tsv
utf8 encode np input txt
$ iconv -c -f utf8 -t utf8 np_txt.tsv |sponge np_txt.tsv
join titles and txt from urls, np
rm embedded newlines np
concat non projid fields
one piece of txt per projid
$ intsv=np.tsv; cat $intsv | tawk '{print $1}'|sed '1d' | sort |uniq|parallel 'txt=$( cat '$intsv' | mawk -F"\t" "{if(\$1 ~ /^{}$/)print \$2}" | tr "\n" " "|sed "s:$:\n:g" ); echo -e {}"\t$txt"'|sponge np.tsv
make iso2 field np
fix selecting allCountries where necessary for get_doc function