File size: 1,568 Bytes
21382b1 635f51d 21382b1 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 |
import pickle
import streamlit as st
model_data = pickle.load(open('gib_model.pki', 'rb'))
import math
import pickle
accepted_chars = 'abcdefghijklmnopqrstuvwxyz '
pos = dict([(char, idx) for idx, char in enumerate(accepted_chars)])
def normalize(line):
""" Return only the subset of chars from accepted_chars.
This helps keep the model relatively small by ignoring punctuation,
infrequently symbols, etc. """
return [c.lower() for c in line if c.lower() in accepted_chars]
def ngram(n, l):
""" Return all n grams from l after normalizing """
filtered = normalize(l)
for start in range(0, len(filtered) - n + 1):
yield ''.join(filtered[start:start + n])
def get_lines():
datasets = ['big.txt']
for filename in datasets:
with open(filename) as fp:
for line in fp:
yield line
def avg_transition_prob(l, log_prob_mat):
""" Return the average transition prob from l through log_prob_mat. """
log_prob = 0.0
transition_ct = 0
for a, b, c in ngram(3, l):
log_prob += log_prob_mat[pos[a]][pos[b]][pos[c]]
transition_ct += 1
# The exponentiation translates from log probs to probs.
return math.exp(log_prob / (transition_ct or 1))
# The exponentiation translates from log probs to probs.
return math.exp(log_prob / (transition_ct or 1))
while True:
l = st.text_area('enter a prospection message')
model_mat = model_data['mat']
threshold = model_data['thresh']
st.write(avg_transition_prob(l, model_mat) > threshold)
|