Fixed inference script bug and made deterministic

streaming_inference.py

@@ -39,11 +39,12 @@ if __name__ == "__main__":
             key = key.replace("lm_head.linear.", "lm_head_linear.")
         model_state_dict[key] = value
     model.load_state_dict(model_state_dict)
+    model.eval()
 
     thread = Thread(
         target=model.generate,
         kwargs=dict(
-            inputs=tokenizer(  # returns a torch dictionary
+            tokenizer(  # returns a torch dictionary
                 "Here is an essay on sea monkeys: ",
                 return_tensors="pt",
                 return_attention_mask=False,