remove data

Scraper.py

@@ -20,4 +20,4 @@ def Scrap(url):
         print("Can't Scrap this article link")
         return None
 
-    return article.text
+    return article

app.py

@@ -1,21 +1,25 @@
 import streamlit as st
 import numpy as np
+import pandas as pd
 import re
 import time
+import os
 
-from transformers import AutoModelForSequenceClassification, AutoTokenizer
+from transformers import AutoModelForSequenceClassification, AutoModel, AutoTokenizer
 from Scraper import Scrap
 
 st.set_page_config(layout="wide")
 
 model_checkpoint = "Rifky/FND"
+data_checkpoint = os.path.join(os.getcwd(), "data", "turnbackhoax.csv")
 label = {0: "valid", 1: "fake"}
 
 @st.cache(show_spinner=False, allow_output_mutation=True)
 def load_model():
     model = AutoModelForSequenceClassification.from_pretrained(model_checkpoint, num_labels=2)
+    base_model = AutoModel.from_pretrained(model_checkpoint)
     tokenizer = AutoTokenizer.from_pretrained(model_checkpoint, fast=True)
-    return model, tokenizer
+    return model, base_model, tokenizer
 
 def sigmoid(x):
     return 1 / (1 + np.exp(-x))
@@ -24,7 +28,8 @@ input_column, reference_column = st.columns(2)
 input_column.write('# Fake News Detection AI')
 
 with st.spinner("Loading Model..."):
-    model, tokenizer = load_model()
+    model, base_model, tokenizer = load_model()
+    data = pd.read_csv(data_checkpoint)
 
 user_input = input_column.text_input("Article url")
 submit = input_column.button("submit")
@@ -43,13 +48,13 @@ if submit:
         text = re.sub(r'\n', ' ', text)
 
         with st.spinner("Computing..."):
-            text = text.split()
-            text_len = len(text)
+            token = text.split()
+            text_len = len(token)
 
             sequences = []
             for i in range(text_len // 512):
-                sequences.append(" ".join(text[i * 512: (i + 1) * 512]))
-            sequences.append(" ".join(text[text_len - (text_len % 512) : text_len]))
+                sequences.append(" ".join(token[i * 512: (i + 1) * 512]))
+            sequences.append(" ".join(token[text_len - (text_len % 512) : text_len]))
             sequences = tokenizer(sequences, max_length=512, truncation=True, padding="max_length", return_tensors='pt')
 
             predictions = model(**sequences)[0].detach().numpy()
@@ -63,4 +68,4 @@ if submit:
             prediction = np.argmax(result, axis=-1)
             input_column.success(f"This news is {label[prediction]}.")
             st.text(f"{int(result[prediction]*100)}% confidence")
-            input_column.progress(result[prediction])
+            input_column.progress(result[prediction])

data/turnbackhoax.json

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:9467d1df1c6b0fe8d927933bf753dd8ca0bbf43ab24c37d216f6c59ca3ca02ff
-size 34821564