public static void main(String[] args)
  {
    Tokenizer tokenizer = new Tokenizer();
    //tokenizer = new Tokenizer("testfile.txt");

    Scanner s = new Scanner( System.in );
    
    while (true)
    {
      String line = s.nextLine();
      List<String> tokens = Tokenizer.tokenize( line );
      if (tokens == null)
        break;

      System.out.println("The tokens are " + tokens );

      System.out.println();

    }
  }