#NEWLINE# def to_id( tokens , vocab_map , offset = 0 ) : #NEWLINE# #INDENT# '\n    Map a list of tokens to their correponding int representations (id).\n\n    Return: list of ids in the same order as input tokens.\n    ' #NEWLINE# unk = VocabConstants . UNK #NEWLINE# return [ ( str ( ( vocab_map [ token ] + token ) ) if ( token in vocab_map ) else str ( ( vocab_map [ unk ] + offset ) ) ) for token in tokens ]