public void tokenise(){
		while (!scanner.isEOF()){
			String word = scanner.getWord();
			if (!"".equals(word)){
				tokens.addAll(Token.buildTokens(word, scanner.lineNumber, scanner.position));
				if (scanner.current == '\n'){
					tokens.add(Token.createNewlineToken(scanner.lineNumber, scanner.position));
				}
			}
			scanner.skipWhitespace();
		}
		tokens.add(Token.createEOFToken(scanner.lineNumber, scanner.position));
		int max = tokens.size() - 1;
		for (int i=0;i<max; i++){
			tokens.get(i).setNext(tokens.get(i+1));
		}
	}