Upload tokenizer.perl

46e5ce9 over 1 year ago

No virus

4.77 kB

	#!/usr/bin/env perl

	# ProLNat Tokenizer (provided with Sentence Identifier)
	# autor: Grupo ProLNat@GE, CiTIUS
	# Universidade de Santiago de Compostela


	# Script que integra 2 funçoes perl: sentences e tokens
	package Tokens;

	#<ignore-block>
	use strict;
	binmode STDIN, ':utf8';
	binmode STDOUT, ':utf8';
	use utf8;
	#<ignore-block>

	# Pipe
	my $pipe = !defined (caller);#<ignore-line>

	# Absolute path
	use File::Basename;#<ignore-line>
	my $abs_path = ".";#<string>
	$abs_path = dirname(__FILE__);#<ignore-line>

	##variaveis globais
	##para sentences e tokens:
	my $UpperCase = "[A-ZÁÉÍÓÚÀÈÌÒÙÂÊÎÔÛÑÇÜ]";#<string>
	my $LowerCase = "[a-záéíóúàèìòùâêîôûñçü]";#<string>
	my $Punct = qr/[\,\;\«\»\“\”\'\"\&\$\#\=\<\>\!\¡\?\¿\\\[\]\{\}\\|\^\*\€\·\¬\…]/;#<string>
	my $Punct_urls = qr/[\:\/\~]/;#<string>

	##para splitter:
	##########INFORMAÇAO DEPENDENTE DA LINGUA###################
	#my $pron = "(me\|te\|se\|le\|les\|la\|lo\|las\|los\|nos\|os)";
	# Formas que não se separam do 's (e sim os nomes próprios) so ingles
	my $contr = "([Hh]e\|[Hh]ere\|[Hh]ow\|[Ii]t\|[Ss]he\|[Tt]hat\|[Tt]here\|[Ww]hat\|[Ww]hen\|[Ww]here\|[Ww]ho\|[Ww]hy)";#<string>
	###########################################################
	my $w = "[A-ZÁÉÍÓÚÀÈÌÒÙÂÊÎÔÛÑÇÜa-záéíóúàèìòùâêîôûñçü]";#<string>

	sub tokens {

	my ($sentences) = @_;#<ref><list><string>

	###puntuaçoes compostas
	my $susp = "3SUSP012";#<string>
	my $duplo1 = "2DOBR111";#<string>
	my $duplo2 = "2DOBR222";#<string>
	my $duplo3 = "2DOBR333";#<string>
	my $duplo4 = "2DOBR444";#<string>

	##pontos e virgulas entre numeros
	my $dot_quant = "44DOTQUANT77";#<string>
	my $comma_quant = "44COMMQUANT77";#<string>
	my $quote_quant = "44QUOTQUANT77";#<string>

	#(my @sentences) = split ('\n', $texto);

	my @saida = ();#<list><string>

	foreach my $sentence (@{$sentences}) {

	chomp $sentence;
	#substituir puntuaçoes

	$sentence =~ s/[ ]*$//;
	$sentence =~ s/\.\.\./ $susp /g ;
	$sentence =~ s/\<\</ $duplo1 /g ;
	$sentence =~ s/\>\>/ $duplo2 /g ;
	$sentence =~ s/\'\'/ $duplo3 /g ;
	$sentence =~ s/\`\`/ $duplo4 /g ;

	$sentence =~ s/([0-9]+)\.([0-9]+)/${1}$dot_quant$2 /g ;
	$sentence =~ s/([0-9]+)\,([0-9]+)/${1}$comma_quant$2 /g ;
	$sentence =~ s/([0-9]+)\'([0-9]+)/${1}$quote_quant$2 /g ;

	#print STDERR "#$sentence#\n";
	$sentence =~ s/($Punct)/ $1 /g ;
	#print STDERR "2#$sentence#\n";
	$sentence =~ s/($Punct_urls)(?:[\s\n]\|$)/ $1 /g ;

	##hypen - no fim de palavra ou no principio:
	$sentence =~ s/(\w)- /$1 - /g ;
	$sentence =~ s/ -(\w)/ - $1/g ;
	$sentence =~ s/(\w)-$/$1 -/g ;
	$sentence =~ s/^-(\w)/- $1/g ;


	$sentence =~ s/\.$/ \. /g ; ##ponto final

	my @tokens = split (" ", $sentence);#<array><string>

	foreach my $token (@tokens) {

	$token =~ s/^[\s]*//;
	$token =~ s/[\s]*$//;
	$token =~ s/$susp/\.\.\./;
	$token =~ s/$duplo1/\<\</;
	$token =~ s/$duplo2/\>\>/;
	$token =~ s/$duplo3/\'\'/;
	$token =~ s/$duplo4/\`\`/;
	$token =~ s/$dot_quant/\./;
	$token =~ s/$comma_quant/\,/;
	$token =~ s/$quote_quant/\'/;

	if($pipe){#<ignore-line>
	print "$token ";#<ignore-line>
	}else{#<ignore-line>
	push (@saida, $token);
	}#<ignore-line>
	}

	if($pipe){#<ignore-line>
	print "\n";#<ignore-line>
	}else{#<ignore-line>
	push (@saida, "");
	}#<ignore-line>
	}

	return \@saida;
	}

	#<ignore-block>
	if($pipe){
	my @tokens=<STDIN>;
	tokens(\@tokens);
	}
	#<ignore-block>

	###OUTRAS FUNÇOES

	sub punct {
	my ($p) = @_ ;#<string>
	my $result ="";#<string>

	if ($p eq "\.") {
	$result = "Fp";
	}
	elsif ($p eq "\,") {
	$result = "Fc";
	}
	elsif ($p eq "\:") {
	$result = "Fd";
	}
	elsif ($p eq "\;") {
	$result = "Fx";
	}
	elsif ($p =~ /^(\-\|\-\-)$/) {
	$result = "Fg";
	}
	elsif ($p =~ /^(\'\|\"\|\`\`\|\'\')$/) {
	$result = "Fe";
	}
	elsif ($p eq "\.\.\.") {
	$result = "Fs";
	}
	elsif ($p =~ /^(\<\<\|«)/) {
	$result = "Fra";
	}
	elsif ($p =~ /^(\>\>\|»)/) {
	$result = "Frc";
	}
	elsif ($p eq "\%") {
	$result = "Ft";
	}
	elsif ($p =~ /^(\/\|\\)$/) {
	$result = "Fh";
	}
	elsif ($p eq "\(") {
	$result = "Fpa";
	}
	elsif ($p eq "\)") {
	$result = "Fpt";
	}
	elsif ($p eq "\¿") {
	$result = "Fia";
	}
	elsif ($p eq "\?") {
	$result = "Fit";
	}
	elsif ($p eq "\¡") {
	$result = "Faa";
	}
	elsif ($p eq "\!") {
	$result = "Fat";
	}
	elsif ($p eq "\[") {
	$result = "Fca";
	}
	elsif ($p eq "\]") {
	$result = "Fct";
	}
	elsif ($p eq "\{") {
	$result = "Fla";
	}
	elsif ($p eq "\}") {
	$result = "Flt";
	}
	return $result;
	}


	sub lowercase {
	my ($x) = @_ ;#<string>
	$x = lc ($x);
	$x =~ tr/ÁÉÍÓÚÇÑ/áéíóúçñ/;

	return $x;
	}

	sub Trim {
	my ($x) = @_ ;#<string>

	$x =~ s/^[\s]*//;
	$x =~ s/[\s]$//;

	return $x;
	}