public List<Token> getBestTokens(Sentence sentence) throws IOException {

		SentenceIterator iterator = sentence.iterator();
		int length = iterator.length();
		char[] surface = sentence.getCharacters();

		// Initialize the Viterbi lattice
		this.bosNode = this.tokenizer.getBOSNode();
		this.eosNode = this.tokenizer.getEOSNode();
		this.endNodeList = new Node[length + 1];
		this.endNodeList[0] = this.bosNode;
		this.endNodeList[length] = null;


		// Look up potential morphemes at each position in the sentence, and
		// join them to the lattice
		while (iterator.hasNextOrigin()) {
			int position = iterator.nextOrigin();
			int base = position - iterator.skippedCharCount();
			if (this.endNodeList[base] != null) {
				Node rNode = lookup(iterator, surface, sentence.getReadingConstraint(position));
				if (rNode != null) {
					calculateConnectionCosts(base, length, rNode);
				}
			}
		}


		// Find the most likely connection from the last position where a
		// morpheme ended to the end-of-string. If there were no morphemes, this
		// will connect to the beginning-of-string node.
		for (int position = length; position >= 0; position--) {
			if (this.endNodeList[position] != null) {
				calculateConnectionCosts(position, length, this.eosNode);
				// Once we have connected the end-of-string node, leave the loop. 
				break;
			}
		}


		// Working backwards from the end-of-string Node, make the forward
		// connections along the most likely path
		Node node = this.eosNode;
		for (Node prevNode; node.prev != null;) {
			prevNode = node.prev;
			prevNode.next = node;
			node = prevNode;
		}


		// Convert to Token list
		String sentenceString = new String(sentence.getCharacters());
		List<Token> tokenList = new ArrayList<Token>();
		node = this.bosNode.next;
		while ((node != null) && (node.next != null)) {
			Token token = new Token(sentenceString, node);
			tokenList.add(token);
			node = node.next;
		}

		return tokenList;

	}