tianyuz commited on
Commit
e50d65b
1 Parent(s): 44d3fde

Update README.md

Browse files
Files changed (1) hide show
  1. README.md +3 -5
README.md CHANGED
@@ -25,13 +25,11 @@ This repository provides a 1.3B-parameter Japanese GPT model. The model was trai
25
 
26
  # How to use the model
27
 
28
- *NOTE:* Use `T5Tokenizer` to initiate the tokenizer.
29
-
30
  ~~~~
31
  import torch
32
- from transformers import T5Tokenizer, AutoModelForCausalLM
33
 
34
- tokenizer = T5Tokenizer.from_pretrained("rinna/japanese-gpt-1b")
35
  model = AutoModelForCausalLM.from_pretrained("rinna/japanese-gpt-1b")
36
 
37
  if torch.cuda.is_available():
@@ -51,7 +49,7 @@ with torch.no_grad():
51
  pad_token_id=tokenizer.pad_token_id,
52
  bos_token_id=tokenizer.bos_token_id,
53
  eos_token_id=tokenizer.eos_token_id,
54
- bad_word_ids=[[tokenizer.unk_token_id]]
55
  )
56
 
57
  output = tokenizer.decode(output_ids.tolist()[0])
 
25
 
26
  # How to use the model
27
 
 
 
28
  ~~~~
29
  import torch
30
+ from transformers import AutoTokenizer, AutoModelForCausalLM
31
 
32
+ tokenizer = AutoTokenizer.from_pretrained("rinna/japanese-gpt-1b", use_fast=False)
33
  model = AutoModelForCausalLM.from_pretrained("rinna/japanese-gpt-1b")
34
 
35
  if torch.cuda.is_available():
 
49
  pad_token_id=tokenizer.pad_token_id,
50
  bos_token_id=tokenizer.bos_token_id,
51
  eos_token_id=tokenizer.eos_token_id,
52
+ bad_words_ids=[[tokenizer.unk_token_id]]
53
  )
54
 
55
  output = tokenizer.decode(output_ids.tolist()[0])