prajdabre commited on
Commit
7e0f28d
1 Parent(s): ca3279f

Update README.md

Browse files
Files changed (1) hide show
  1. README.md +3 -3
README.md CHANGED
@@ -21,9 +21,9 @@ pad_id = tokenizer._convert_token_to_id_with_added_voc("<pad>")
21
  # To get lang_id use any of ['<2as>', '<2bn>', '<2en>', '<2gu>', '<2hi>', '<2kn>', '<2ml>', '<2mr>', '<2or>', '<2pa>', '<2ta>', '<2te>']
22
 
23
  # First tokenize the input and outputs. The format below is how IndicBART-XXEN was trained so the input should be "Sentence </s> <2xx>" where xx is the language code. Similarly, the output should be "<2yy> Sentence </s>".
24
- inp = tokenizer("मैं एक लड़का हूँ </s> <2hi>", add_special_tokens=False, return_tensors="pt", padding=True).input_ids # tensor([[ 466, 1981, 80, 25573, 64001, 64004]])
25
 
26
- out = tokenizer("<2en> I am a boy </s>", add_special_tokens=False, return_tensors="pt", padding=True).input_ids # tensor([[64006, 942, 43, 32720, 8384, 64001]])
27
 
28
  model_outputs=model(input_ids=inp, decoder_input_ids=out[:,0:-1], labels=out[:,1:])
29
 
@@ -37,7 +37,7 @@ model_outputs.logits
37
 
38
  model.eval() # Set dropouts to zero
39
 
40
- model_output=model.generate(inp, use_cache=True, num_beams=4, max_length=20, min_length=1, early_stopping=True, pad_token_id=pad_id, bos_token_id=bos_id, eos_token_id=eos_id, decoder_start_token_id=tokenizer._convert_token_to_id_with_added_voc("<2hi>"))
41
 
42
 
43
  # Decode to get output strings
 
21
  # To get lang_id use any of ['<2as>', '<2bn>', '<2en>', '<2gu>', '<2hi>', '<2kn>', '<2ml>', '<2mr>', '<2or>', '<2pa>', '<2ta>', '<2te>']
22
 
23
  # First tokenize the input and outputs. The format below is how IndicBART-XXEN was trained so the input should be "Sentence </s> <2xx>" where xx is the language code. Similarly, the output should be "<2yy> Sentence </s>".
24
+ inp = tokenizer("मैं एक लड़का हूँ </s> <2hi>", add_special_tokens=False, return_tensors="pt", padding=True).input_ids
25
 
26
+ out = tokenizer("<2en> I am a boy </s>", add_special_tokens=False, return_tensors="pt", padding=True).input_ids
27
 
28
  model_outputs=model(input_ids=inp, decoder_input_ids=out[:,0:-1], labels=out[:,1:])
29
 
 
37
 
38
  model.eval() # Set dropouts to zero
39
 
40
+ model_output=model.generate(inp, use_cache=True, num_beams=4, max_length=20, min_length=1, early_stopping=True, pad_token_id=pad_id, bos_token_id=bos_id, eos_token_id=eos_id, decoder_start_token_id=tokenizer._convert_token_to_id_with_added_voc("<2en>"))
41
 
42
 
43
  # Decode to get output strings