Changed the model path

.ipynb_checkpoints/app-checkpoint.py

@@ -20,9 +20,9 @@ def generate_kws(context,num_kw, kw_ngs):
             kw_ngs=int(kw_ngs)
         except ValueError:
             kw_ngs=None
-        csv_encoded.truncate(0)
-        writer.writerow([context])
-        context=csv_encoded.getvalue()
+        #csv_encoded.truncate(0)
+        #writer.writerow([context])
+        #context=csv_encoded.getvalue()
         return kw_ex.extract(context, num_kw, kw_ngs) or ""
     else:
         raise gr.Error("Please enter text in inputbox!!!!")

.ipynb_checkpoints/kwextractor-checkpoint.py

@@ -26,15 +26,26 @@ class KeyWordExtractor():
         
         #self.kw_model.to(self.device)
         
+    def _extract_by_paragraph(self, ctxt, nkws=None, max_kw_ngs=None):
+        paragraphs=map(str.strip,ctxt.split("\n"))
+        kws=[]
+        for paragraph in paragraphs:
+            if paragraph:
+                kws.extend(self.kw_model.extract_keywords(paragraph, keyphrase_ngram_range=(1, max_kw_ngs),
+                                                          top_n=nkws,
+                                     #use_maxsum=True,nr_candidates=20, top_n=5,
+                                     #use_mmr=True, 
+                                          #diversity=0.8,
+                                                          stop_words=None)
+                          )
+         
+        return sorted(kws, key=lambda x: x[1],reverse=True)[:nkws]
+        
     def extract(self, ctxt, nkws=None, max_kw_ngs=None):
         nkws= nkws if nkws is not None else self.NKW
         max_kw_ngs=max_kw_ngs if max_kw_ngs is not None else self.MAX_KW_NGS
         
-        kw=self.kw_model.extract_keywords(ctxt, keyphrase_ngram_range=(1, max_kw_ngs),
-                                          top_n=nkws,
-                                     #use_maxsum=True,nr_candidates=20, top_n=5,
-                                     #use_mmr=True, 
-                                          #diversity=0.8,
-                                     stop_words=None)
+        #Since we are taking only 512 tokens, let's do by paragraph
+        kw=self._extract_by_paragraph(ctxt,nkw,max_kw_ngs)
         return ", ".join(w for w,_ in kw)
     

app.py

@@ -20,9 +20,9 @@ def generate_kws(context,num_kw, kw_ngs):
             kw_ngs=int(kw_ngs)
         except ValueError:
             kw_ngs=None
-        csv_encoded.truncate(0)
-        writer.writerow([context])
-        context=csv_encoded.getvalue()
+        #csv_encoded.truncate(0)
+        #writer.writerow([context])
+        #context=csv_encoded.getvalue()
         return kw_ex.extract(context, num_kw, kw_ngs) or ""
     else:
         raise gr.Error("Please enter text in inputbox!!!!")

kwextractor.py

@@ -26,15 +26,26 @@ class KeyWordExtractor():
         
         #self.kw_model.to(self.device)
         
+    def _extract_by_paragraph(self, ctxt, nkws=None, max_kw_ngs=None):
+        paragraphs=map(str.strip,ctxt.split("\n"))
+        kws=[]
+        for paragraph in paragraphs:
+            if paragraph:
+                kws.extend(self.kw_model.extract_keywords(paragraph, keyphrase_ngram_range=(1, max_kw_ngs),
+                                                          top_n=nkws,
+                                     #use_maxsum=True,nr_candidates=20, top_n=5,
+                                     #use_mmr=True, 
+                                          #diversity=0.8,
+                                                          stop_words=None)
+                          )
+         
+        return sorted(kws, key=lambda x: x[1],reverse=True)[:nkws]
+        
     def extract(self, ctxt, nkws=None, max_kw_ngs=None):
         nkws= nkws if nkws is not None else self.NKW
         max_kw_ngs=max_kw_ngs if max_kw_ngs is not None else self.MAX_KW_NGS
         
-        kw=self.kw_model.extract_keywords(ctxt, keyphrase_ngram_range=(1, max_kw_ngs),
-                                          top_n=nkws,
-                                     #use_maxsum=True,nr_candidates=20, top_n=5,
-                                     #use_mmr=True, 
-                                          #diversity=0.8,
-                                     stop_words=None)
+        #Since we are taking only 512 tokens, let's do by paragraph
+        kw=self._extract_by_paragraph(ctxt,nkw,max_kw_ngs)
         return ", ".join(w for w,_ in kw)