Kaggle competition: Night at Cameo
<SOS> metric: map@{k} Kaggle data source: night-at-cameo<train_model>
%matplotlib inline
warnings.simplefilter(action="ignore", category=FutureWarning )
sub = sum(
 map(
 lambda i: train_and_inference(splits, i)[0].set_index('recording_id'),
range(len(splits)) 
)
 ).reset_index()<save_to_csv>
shutil.copyfile(src=".. /input/redcarpet.py", dst=".. /working/redcarpet.py")
sub.to_csv("submission.csv", index=False )<load_pretrained>
item_file = ".. /input/talent.pkl"
item_records, COLUMN_LABELS, READABLE_LABELS, ATTRIBUTES = pickle.load(open(item_file, "rb"))
item_df = pd.DataFrame(item_records)[ATTRIBUTES + COLUMN_LABELS].fillna(value=0)
 ITEM_NAMES = item_df["name"].values
ITEM_IDS = item_df["id"].values
s_items = mat_to_sets(item_df[COLUMN_LABELS].values)
 assert len(item_df)== len(s_items), "Item matrix is not the same length as item category set list."
print("Talent:", len(item_df))
print("Categories:", len(COLUMN_LABELS))
item_df.head()
nltk.download('stopwords')
 nltk.download('punkt')
 nltk.download('averaged_perceptron_tagger')
 nltk.download('wordnet')
 !pip install https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-2.2.5/en_core_web_sm-2.2.5.tar.gz
stop = set(stopwords.words('english'))
plt.style.use('fivethirtyeight')
 sns.set(font_scale=1.5)
 pd.options.display.max_columns = 250
pd.options.display.max_rows = 250
warnings.filterwarnings('ignore')
 seed_val = 42
random.seed(seed_val)
 np.random.seed(seed_val)
 torch.manual_seed(seed_val)
 torch.cuda.manual_seed_all(seed_val )<load_from_csv>
csr_train, csr_test, csr_input, csr_hidden = pickle.load(open(".. /input/train_test_mat.pkl", "rb"))
m_split = [np.array(csr.todense())for csr in [csr_train, csr_test, csr_input, csr_hidden]]
m_train, m_test, m_input, m_hidden = m_split
s_train, s_test, s_input, s_hidden = pickle.load(open(".. /input/train_test_set.pkl", "rb"))
assert len(m_train)== len(s_train), "Train matrix is not the same length as train sets."
assert len(m_test)== len(s_test), "Test matrix is not the same length as test sets."
assert len(m_input)== len(s_input), "Input matrix is not the same length as input sets."
assert len(m_hidden)== len(s_hidden), "Hidden matrix is not the same length as hidden sets."
print("Train Users", len(m_train))
print("Test Users", len(m_test))
print("Minimum Test Items per User:", min(m_test.sum(axis=1)))
 print("Minimum Input Items per User:", min(m_input.sum(axis=1)))
 print("Minimum Hidden Items per User:", min(m_hidden.sum(axis=1)))
 like_df = pd.DataFrame(m_train, columns=ITEM_NAMES)
 like_df.head()
trainv = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 testv = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv' )<define_variables>
from redcarpet import mapk_score, uhr_score
def remove_URL(text):
url = re.compile(r'https?://\S+|www\.\S+')
 return url.sub(r'', text)
 def remove_emoji(text):
emoji_pattern = re.compile(
 '['
u'\U0001F600-\U0001F64F'
u'\U0001F300-\U0001F5FF'
u'\U0001F680-\U0001F6FF'
u'\U0001F1E0-\U0001F1FF'
u'\U00002702-\U000027B0'
u'\U000024C2-\U0001F251'
']+',
flags=re.UNICODE)
 return emoji_pattern.sub(r'', text)
 def remove_html(text):
html = re.compile(r'<.*?>|&([a-z0-9]+|
return re.sub(html, '', text)
 def remove_punct(text):
table = str.maketrans('', '', string.punctuation)
 return text.translate(table)
 trainv['text_clean'] = trainv['text'].apply(lambda x: remove_URL(x))
trainv['text_clean'] = trainv['text_clean'].apply(lambda x: remove_emoji(x))
trainv['text_clean'] = trainv['text_clean'].apply(lambda x: remove_html(x))
trainv['text_clean'] = trainv['text_clean'].apply(lambda x: remove_punct(x))<feature_engineering>
from redcarpet import jaccard_sim, cosine_sim
trainv['tokenized'] = trainv['text_clean'].apply(word_tokenize)
 trainv.head()<feature_engineering>
from redcarpet import collaborative_filter, content_filter, weighted_hybrid
trainv['lower'] = trainv['tokenized'].apply(
 lambda x: [word.lower() for word in x])
 trainv.head()<feature_engineering>
collab_jac10 = collaborative_filter(s_train, s_input, j=30, sim_fn=jaccard_sim, threshold=0.05, k=10 )
trainv['stopwords_removed'] = trainv['lower'].apply(
 lambda x: [word for word in x if word not in stop])
 trainv.head()<feature_engineering>
from redcarpet import get_recs
from redcarpet import show_user_recs, show_item_recs, show_user_detail
from redcarpet import show_apk_dist, show_hit_dist, show_score_dist
trainv['pos_tags'] = trainv['stopwords_removed'].apply(nltk.tag.pos_tag)
 trainv.head()<feature_engineering>
k_top = 10
def get_wordnet_pos(tag):
if tag.startswith('J'):
return wordnet.ADJ
elif tag.startswith('V'):
return wordnet.VERB
elif tag.startswith('N'):
return wordnet.NOUN
elif tag.startswith('R'):
return wordnet.ADV
else:
return wordnet.NOUN
trainv['wordnet_pos'] = trainv['pos_tags'].apply(
 lambda x: [(word, get_wordnet_pos(pos_tag)) for(word, pos_tag)in x])
 trainv.head()<feature_engineering>
print("Model: Collaborative Filtering with Jacccard Similarity(j=10)")
 collab_jac10 = collaborative_filter(s_train, s_input, sim_fn=jaccard_sim, j=50, threshold=0.005, k=k_top)
 print("MAP = {0:.3f}".format(mapk_score(s_hidden, get_recs(collab_jac10), k=k_top)))
 print("UHR = {0:.3f}".format(uhr_score(s_hidden, get_recs(collab_jac10), k=k_top)) )
wnl = WordNetLemmatizer()
trainv['lemmatized'] = trainv['wordnet_pos'].apply(
 lambda x: [wnl.lemmatize(word, tag)for word, tag in x])
 trainv['lemmatized'] = trainv['lemmatized'].apply(
 lambda x: [word for word in x if word not in stop])
 trainv['lemma_str'] = [' '.join(map(str, l)) for l in trainv['lemmatized']]
trainv.head()<filter>
print("Model: Collaborative Filtering with Jacccard Similarity(j=10)")
 collab_cos10 = collaborative_filter(s_train, s_input, sim_fn=cosine_sim, j=41, threshold=0.08, k=k_top)
 print("MAP = {0:.3f}".format(mapk_score(s_hidden, get_recs(collab_jac10), k=k_top)))
 print("UHR = {0:.3f}".format(uhr_score(s_hidden, get_recs(collab_jac10), k=k_top)) )
lis = [
trainv[trainv['target'] == 0]['lemma_str'],
trainv[trainv['target'] == 1]['lemma_str']
]<prepare_x_and_y>
content_jac10 = content_filter(s_items, s_input, sim_fn=jaccard_sim, threshold=0.2, k=k_top)
 print("MAP = {0:.3f}".format(mapk_score(s_hidden, get_recs(content_jac10), k=k_top)))
 print("UHR = {0:.3f}".format(uhr_score(s_hidden, get_recs(content_jac10), k=k_top)) )
fig, axes = plt.subplots(1, 2, figsize=(18, 8))
axes = axes.flatten()
for i, j in zip(lis, axes):
try:
new = i.str.split()
new = new.values.tolist()
corpus = [word.lower() for i in new for word in i]
dic = defaultdict(int)
 for word in corpus:
if word in stop:
dic[word] += 1
top = sorted(dic.items() , key=lambda x: x[1], reverse=True)[:15]
x, y = zip(*top)
 df = pd.DataFrame([x, y] ).T
df = df.rename(columns={0: 'Stopword', 1: 'Count'})
 sns.barplot(x='Count', y='Stopword', data=df, palette='plasma', ax=j)
 plt.tight_layout()
except:
plt.close()
print('No stopwords left in texts.')
 break<statistical_test>
print("Model: Hybrid Collaborative Filtering")
 print("Similarity: Hybrid(0.2 * Jaccard + 0.8 * Cosine)")
 collab_hybrid = weighted_hybrid([ 
(collab_jac10, 0.5), 
(collab_cos10, 0.5)
 ])
 print("MAP = {0:.3f}".format(mapk_score(s_hidden, get_recs(collab_hybrid), k=k_top)))
 print("UHR = {0:.3f}".format(uhr_score(s_hidden, get_recs(collab_hybrid), k=k_top)) )
def ngrams(n, title):

 fig, axes = plt.subplots(1, 2, figsize=(18, 8))
axes = axes.flatten()
for i, j in zip(lis, axes):
new = i.str.split()
new = new.values.tolist()
corpus = [word for i in new for word in i]
def _get_top_ngram(corpus, n=None):
vec = CountVectorizer(ngram_range=(n, n),
max_df=0.9,
stop_words='english' ).fit(corpus)
 bag_of_words = vec.transform(corpus)
 sum_words = bag_of_words.sum(axis=0)
 words_freq = [(word, sum_words[0, idx])
 for word, idx in vec.vocabulary_.items() ]
words_freq = sorted(words_freq, key=lambda x: x[1], reverse=True)
 return words_freq[:15]
top_n_bigrams = _get_top_ngram(i, n)[:15]
x, y = map(list, zip(*top_n_bigrams))
sns.barplot(x=y, y=x, palette='plasma', ax=j)
 axes[0].set_title('Non Disaster Tweets')
 axes[1].set_title('Disaster Tweets')
 axes[0].set_xlabel('Count')
 axes[0].set_ylabel('Words')
 axes[1].set_xlabel('Count')
 axes[1].set_ylabel('Words')
 fig.suptitle(title, fontsize=24, va='baseline')
 plt.tight_layout()<feature_engineering>
print("Model: Collaborative Filtering with Cosine Similarity(j=10)")
 collab_cos10 = collaborative_filter(s_train, s_input, sim_fn=cosine_sim, j=10, k=k_top)
 print("MAP = {0:.3f}".format(mapk_score(s_hidden, get_recs(collab_cos10), k=k_top)))
 print("UHR = {0:.3f}".format(uhr_score(s_hidden, get_recs(collab_cos10), k=k_top)) )
def display_topics(text, no_top_words, topic):

 no_top_words = no_top_words
tfidf_vectorizer = TfidfVectorizer(
 max_df=0.90, min_df=25, max_features=5000, use_idf=True)
 tfidf = tfidf_vectorizer.fit_transform(text)
 tfidf_feature_names = tfidf_vectorizer.get_feature_names()
doc_term_matrix_tfidf = pd.DataFrame(
 tfidf.toarray() , columns=list(tfidf_feature_names))
nmf = NMF(n_components=10, random_state=0,
alpha=.1, init='nndsvd' ).fit(tfidf)
 print(topic)
 for topic_idx, topic in enumerate(nmf.components_):
print('Topic %d:' %(topic_idx+1))
print(' '.join([tfidf_feature_names[i]
for i in topic.argsort() [:-no_top_words - 1:-1]]))
display_topics(lis[0], 10, 'Non Disaster Topics
' )<feature_engineering>
results = [ 
(collab_jac10, "Jaccard(j=10)"), 
(collab_cos10, "Cosine(j=10)")
 ]
mask = np.array(Image.open('/kaggle/input/twittermaskn/twittermask.png'))
mask[mask.sum(axis=2)== 0] = 255<load_pretrained>
show_hit_dist(s_hidden, results, k=k_top )
nlp = en_core_web_sm.load()<set_options>
print("Model: Hybrid Collaborative Filtering")
 print("Similarity: Hybrid(0.2 * Jaccard + 0.8 * Cosine)")
 collab_hybrid = weighted_hybrid([ 
(collab_jac10, 0.2), 
(collab_cos10, 0.8)
 ])
 print("MAP = {0:.3f}".format(mapk_score(s_hidden, get_recs(collab_hybrid), k=k_top)))
 print("UHR = {0:.3f}".format(uhr_score(s_hidden, get_recs(collab_hybrid), k=k_top)) )
if torch.cuda.is_available() :
device = torch.device('cuda')
 print('There are %d GPU(s)available.' % torch.cuda.device_count())
 print('We will use the GPU:', torch.cuda.get_device_name(0))
else:
print('No GPU available, using the CPU instead.')
 device = torch.device('cpu' )<load_from_csv>
from redcarpet import write_kaggle_recs, download_kaggle_recs
train = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 test = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
 print(f'Number of training tweets: {train.shape[0]}
')
 print(f'Number of training tweets: {test.shape[0]}
')
 display(train.sample(10))<concatenate>
s_hold_input = pickle.load(open(".. /input/hold_set.pkl", "rb"))
print("Hold Out Set: N = {}".format(len(s_hold_input)))
 s_all_input = s_input + s_hold_input
print("All Input: N = {}".format(len(s_all_input)) )
labels = train['target'].values
idx = len(labels)
 combined = pd.concat([train, test])
 combined = combined.text.values<load_pretrained>
print("Final Model")
 print("Strategy: Collaborative")
 print("Similarity: Cosine(j=10)")
 final_scores = collaborative_filter(s_train, s_all_input, sim_fn=cosine_sim, j=41, threshold=0.08, k=10)
 final_recs = get_recs(final_scores )
<string_transform><EOS> 
outfile = "kaggle_submission_hybrid_collab.csv"
n_lines = write_kaggle_recs(final_recs, outfile)
 print("Wrote predictions for {} users to {}.".format(n_lines, outfile))
download_kaggle_recs(final_recs, outfile )