p-christ commited on
Commit
af0421d
1 Parent(s): 827cc47

Update pipeline.py

Browse files
Files changed (1) hide show
  1. pipeline.py +2 -0
pipeline.py CHANGED
@@ -2,6 +2,7 @@ from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
2
  from typing import Dict, List, Any
3
  import itertools
4
  from nltk import sent_tokenize
 
5
 
6
  class PreTrainedPipeline():
7
  def __init__(self, path=""):
@@ -9,6 +10,7 @@ class PreTrainedPipeline():
9
  # Preload all the elements you are going to need at inference.
10
  # For instance your model, processors, tokenizer that might be needed.
11
  # This function is only called once, so do all the heavy processing I/O here"""
 
12
  self.model = AutoModelForSeq2SeqLM.from_pretrained(path)
13
  self.tokenizer = AutoTokenizer.from_pretrained(path)
14
 
 
2
  from typing import Dict, List, Any
3
  import itertools
4
  from nltk import sent_tokenize
5
+ import nltk
6
 
7
  class PreTrainedPipeline():
8
  def __init__(self, path=""):
 
10
  # Preload all the elements you are going to need at inference.
11
  # For instance your model, processors, tokenizer that might be needed.
12
  # This function is only called once, so do all the heavy processing I/O here"""
13
+ nltk.download('punkt')
14
  self.model = AutoModelForSeq2SeqLM.from_pretrained(path)
15
  self.tokenizer = AutoTokenizer.from_pretrained(path)
16