File size: 2,242 Bytes
700431a de435e1 700431a 7532efb de435e1 700431a bb8ddb0 700431a 84938ff 700431a 84938ff 700431a b2de734 700431a 1833f3a c356db2 700431a 84938ff 700431a 84938ff 700431a 84938ff c356db2 b2de734 84938ff 700431a 84938ff 700431a |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 |
from transformers import PreTrainedModel, PretrainedConfig
from tensorflow.keras.models import load_model
from tensorflow.keras.preprocessing.text import tokenizer_from_json
from tensorflow.keras.preprocessing.sequence import pad_sequences
import numpy as np
import json
class NewsClassifierConfig(PretrainedConfig):
model_type = "text_classifier"
def __init__(
self,
max_length=41, # Modified to match model input shape
vocab_size=74934, # Modified based on embedding layer size
embedding_dim=128, # Added to match model architecture
hidden_size=64, # Matches final LSTM layer
num_labels=2,
**kwargs
):
self.max_length = max_length
self.vocab_size = vocab_size
self.embedding_dim = embedding_dim
self.hidden_size = hidden_size
self.num_labels = num_labels
super().__init__(**kwargs)
class NewsClassifier(PreTrainedModel):
config_class = NewsClassifierConfig
base_model_prefix = "text_classifier"
def __init__(self, config):
super().__init__(config)
self.model = None
self.tokenizer = None
def post_init(self):
"""Load model and tokenizer after initialization"""
self.model = load_model('news_classifier.h5')
with open('tokenizer.json', 'r') as f:
tokenizer_data = json.load(f)
self.tokenizer = tokenizer_from_json(tokenizer_data)
def forward(self, text_input):
if not self.model or not self.tokenizer:
self.post_init()
if isinstance(text_input, str):
text_input = [text_input]
sequences = self.tokenizer.texts_to_sequences(text_input)
padded = pad_sequences(sequences, maxlen=self.config.max_length)
predictions = self.model.predict(padded, verbose=0)
results = []
for pred in predictions:
score = float(pred[1])
label = "foxnews" if score > 0.5 else "nbc"
results.append({
"label": label,
"score": score if label == "foxnews" else 1 - score
})
return results[0] if len(text_input) == 1 else results |