update readme

README.md

@@ -17,20 +17,39 @@ RedPajama-Base-INCITE-6.9B-v1, is a large transformer-based language model devel
 
 # Quick Start
 
+Please note that the model requires `transformers` version >= 4.25.1.
+
 ## GPU Inference
 
 This requires a GPU with 16GB memory.
+
 ```python
+import torch
+import transformers
 from transformers import AutoTokenizer, AutoModelForCausalLM
+
+MIN_TRANSFORMERS_VERSION = '4.25.1'
+
+# check transformers version
+assert transformers.__version__ >= MIN_TRANSFORMERS_VERSION, f'Please upgrade transformers to version {MIN_TRANSFORMERS_VERSION} or higher.'
+
 # init
 tokenizer = AutoTokenizer.from_pretrained("togethercomputer/RedPajama-Base-INCITE-6.9B-v1")
 model = AutoModelForCausalLM.from_pretrained("togethercomputer/RedPajama-Base-INCITE-6.9B-v1", torch_dtype=torch.float16)
 model = model.to('cuda:0')
 # infer
-inputs = tokenizer("Hello", return_tensors='pt').to(model.device)
-outputs = model.generate(**inputs, max_new_tokens=10, do_sample=True, temperature=0.8)
-output_str = tokenizer.decode(outputs[0])
+prompt = "Alan Turing is"
+inputs = tokenizer(prompt, return_tensors='pt').to(model.device)
+input_length = inputs.input_ids.shape[1]
+outputs = model.generate(
+    **inputs, max_new_tokens=128, do_sample=True, temperature=0.7, top_p=0.7, top_k=50, return_dict_in_generate=True
+)
+token = outputs.sequences[0, input_length:]
+output_str = tokenizer.decode(token)
 print(output_str)
+"""
+widely considered to be the father of modern computer science and artificial intelligence. He was a brilliant mathematician and cryptographer, who worked for the British government during World War II. He was instrumental in breaking the German Enigma code, and is credited with helping to shorten the war by two years...
+"""
 ```
 
 ## GPU Inference in Int8
@@ -38,31 +57,64 @@ print(output_str)
 This requires a GPU with 12GB memory.
 
 ```python
+import torch
+import transformers
 from transformers import AutoTokenizer, AutoModelForCausalLM
+
+MIN_TRANSFORMERS_VERSION = '4.25.1'
+
+# check transformers version
+assert transformers.__version__ >= MIN_TRANSFORMERS_VERSION, f'Please upgrade transformers to version {MIN_TRANSFORMERS_VERSION} or higher.'
+
 # init
 tokenizer = AutoTokenizer.from_pretrained("togethercomputer/RedPajama-Base-INCITE-6.9B-v1")
-model = AutoModelForCausalLM.from_pretrained("togethercomputer/RedPajama-Base-INCITE-6.9B-v1", device_map="auto", load_in_8bit=True)
+model = AutoModelForCausalLM.from_pretrained("togethercomputer/RedPajama-Base-INCITE-6.9B-v1", device_map='auto', torch_dtype=torch.float16, load_in_8bit=True)
+
 # infer
-inputs = tokenizer("Hello", return_tensors='pt').to(model.device)
-outputs = model.generate(**inputs, max_new_tokens=10, do_sample=True, temperature=0.8)
-output_str = tokenizer.decode(outputs[0])
+prompt = "Alan Turing is"
+inputs = tokenizer(prompt, return_tensors='pt').to(model.device)
+input_length = inputs.input_ids.shape[1]
+outputs = model.generate(
+    **inputs, max_new_tokens=128, do_sample=True, temperature=0.7, top_p=0.7, top_k=50, return_dict_in_generate=True
+)
+token = outputs.sequences[0, input_length:]
+output_str = tokenizer.decode(token)
 print(output_str)
-```
+"""
+a very well-known name in the world of computer science. It is named after the mathematician Alan Turing. He is famous for his work on the Enigma machine, which was used by the Germans during World War II....
+"""```
 
 ## CPU Inference
 
 ```python
+import torch
+import transformers
 from transformers import AutoTokenizer, AutoModelForCausalLM
+
+MIN_TRANSFORMERS_VERSION = '4.25.1'
+
+# check transformers version
+assert transformers.__version__ >= MIN_TRANSFORMERS_VERSION, f'Please upgrade transformers to version {MIN_TRANSFORMERS_VERSION} or higher.'
+
 # init
 tokenizer = AutoTokenizer.from_pretrained("togethercomputer/RedPajama-Base-INCITE-6.9B-v1")
 model = AutoModelForCausalLM.from_pretrained("togethercomputer/RedPajama-Base-INCITE-6.9B-v1", torch_dtype=torch.bfloat16)
 # infer
-inputs = tokenizer("<human>: Hello!\n<bot>:", return_tensors='pt').to(model.device)
-outputs = model.generate(**inputs, max_new_tokens=10, do_sample=True, temperature=0.8)
-output_str = tokenizer.decode(outputs[0])
+prompt = "Alan Turing is"
+inputs = tokenizer(prompt, return_tensors='pt').to(model.device)
+input_length = inputs.input_ids.shape[1]
+outputs = model.generate(
+    **inputs, max_new_tokens=128, do_sample=True, temperature=0.7, top_p=0.7, top_k=50, return_dict_in_generate=True
+)
+token = outputs.sequences[0, input_length:]
+output_str = tokenizer.decode(token)
 print(output_str)
+"""
+one of the most important figures in the history of computing. He is best known for his work on the development of the modern computer and for his code-breaking work during World War II. He was also a brilliant mathematician and philosopher.
+"""
 ```
 
+Please note that since `LayerNormKernelImpl` is not implemented in fp16 for CPU, we use `bfloat16` for CPU inference.
 
 # Uses