#NEWLINE# def get_tokens( self , data ) : #NEWLINE# #INDENT# '\n        Acts as a lexer: given an input text, converts a sequence of characters into a sequence of\n        tokens that represents the ramifications of the nested BBCode tags.\n        Each token embeds the following data:\n\n            Token type\n                It can be: START_TAG, END_TAG, DATA, NEWLINE\n            Tag name\n                The name of the tokenized tag if a tag is considered, defaults to None\n            Tag option\n                The content of the tag option if available, defaults to None\n            Token text\n                The original text of the token\n        ' #NEWLINE# tokens = [ ] #NEWLINE# pos = tag_start = tag_end = 0 #NEWLINE# if self . normalize_newlines : #NEWLINE# #INDENT# data = data . replace ( '\r\n' , '\n' ) . replace ( '\r' , '\n' ) #NEWLINE# #UNINDENT# while ( pos < len ( data ) ) : #NEWLINE# #INDENT# tag_start = data . find ( self . _TAG_OPENING , pos ) #NEWLINE# pos_diff = ( tag_start - pos ) #NEWLINE# if ( pos_diff >= 0 ) : #NEWLINE# #INDENT# if pos_diff : #NEWLINE# #INDENT# tokens . extend ( self . _get_textual_tokens ( data [ pos : tag_start ] ) ) #NEWLINE# #UNINDENT# tag_end = data . find ( self . _TAG_ENDING , tag_start ) #NEWLINE# new_tag_start = data . find ( self . _TAG_OPENING , ( tag_start + len ( self . _TAG_OPENING ) ) ) #NEWLINE# if ( ( new_tag_start > 0 ) and ( new_tag_start < tag_end ) ) : #NEWLINE# #INDENT# tokens . extend ( self . _get_textual_tokens ( data [ tag_start : new_tag_start ] ) ) #NEWLINE# pos = new_tag_start #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# if ( tag_end > tag_start ) : #NEWLINE# #INDENT# tag = data [ tag_start : ( tag_end + len ( self . _TAG_ENDING ) ) ] #NEWLINE# ( valid , tag_name , closing , option ) = self . _parse_tag ( tag ) #NEWLINE# if ( valid and ( tag_name in self . bbcodes ) ) : #NEWLINE# #INDENT# if closing : #NEWLINE# #INDENT# tokens . append ( BBCodeToken ( BBCodeToken . TK_END_TAG , tag_name , None , tag ) ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# tokens . append ( BBCodeToken ( BBCodeToken . TK_START_TAG , tag_name , option , tag ) ) #NEWLINE# #UNINDENT# #UNINDENT# else : #NEWLINE# #INDENT# tokens . extend ( self . _get_textual_tokens ( tag ) ) #NEWLINE# #UNINDENT# pos = ( tag_end + len ( self . _TAG_ENDING ) ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# break #NEWLINE# #UNINDENT# #UNINDENT# #UNINDENT# else : #NEWLINE# #INDENT# break #NEWLINE# #UNINDENT# #UNINDENT# if ( pos < len ( data ) ) : #NEWLINE# #INDENT# tokens . extend ( self . _get_textual_tokens ( data [ pos : ] ) ) #NEWLINE# #UNINDENT# return tokens