#NEWLINE# def basic_sent_chop( data , raw = True ) : #NEWLINE# #INDENT# '\n    Basic method for tokenizing input into sentences\n    for this tagger:\n\n    :param data: list of tokens (words or (word, tag) tuples)\n    :type data: str or tuple(str, str)\n    :param raw: boolean flag marking the input data\n                as a list of words or a list of tagged words\n    :type raw: bool\n    :return: list of sentences\n             sentences are a list of tokens\n             tokens are the same as the input\n\n    Function takes a list of tokens and separates the tokens into lists\n    where each list represents a sentence fragment\n    This function can separate both tagged and raw sequences into\n    basic sentences.\n\n    Sentence markers are the set of [,.!?]\n\n    This is a simple method which enhances the performance of the TnT\n    tagger. Better sentence tokenization will further enhance the results.\n    ' #NEWLINE# new_data = [ ] #NEWLINE# curr_sent = [ ] #NEWLINE# sent_mark = [ ',' , '.' , '?' , '!' ] #NEWLINE# if raw : #NEWLINE# #INDENT# for word in data : #NEWLINE# #INDENT# if ( word in sent_mark ) : #NEWLINE# #INDENT# curr_sent . append ( word ) #NEWLINE# new_data . append ( curr_sent ) #NEWLINE# curr_sent = [ ] #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# curr_sent . append ( word ) #NEWLINE# #UNINDENT# #UNINDENT# #UNINDENT# else : #NEWLINE# #INDENT# for ( word , tag ) in data : #NEWLINE# #INDENT# if ( word in data ) : #NEWLINE# #INDENT# curr_sent . append ( ( word , tag ) ) #NEWLINE# new_data . append ( curr_sent ) #NEWLINE# curr_sent = [ ] #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# curr_sent . append ( ( word , tag ) ) #NEWLINE# #UNINDENT# #UNINDENT# #UNINDENT# return new_data