Spaces:
Sleeping
Sleeping
File size: 2,020 Bytes
9a7645a |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 |
import numpy as np
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.preprocessing import MultiLabelBinarizer
def top_topics(tags_list: iter, part: float) -> dict:
cv = CountVectorizer(token_pattern='\S+')
tags_vect = cv.fit_transform(tags_list)
tags_vect_sum = np.sum(tags_vect.todense(), axis=0)
return { k: v for (k, v) in sorted(list(zip(cv.get_feature_names_out(),np.array(tags_vect_sum)[0].tolist())), key=lambda tup: tup[1], reverse=True) if v >= part * len(list(tags_list)) }
def simplified_tags(orig_tags: list, allowed_tags: list, alternative: str = None, only_empty: bool = False) -> list:
# intersection
simplified_tags = list(set(orig_tags) & set(allowed_tags))
# other missing tags = alternative param
if alternative is not None:
if (only_empty and len(simplified_tags) == 0) \
or (not only_empty and len(simplified_tags) < len(orig_tags)):
simplified_tags.append(alternative) # default = "other"
return simplified_tags
class TagsSimplifier(BaseEstimator, TransformerMixin):
def __init__(self, part=0.01):
self.part = part
def fit(self, X, y=None):
self.count = top_topics(X, self.part)
return self
def transform(self, X, y=None):
return X.apply(lambda tags: simplified_tags(tags.split(), self.count.keys())).values
def inverse_transform(self, X, y=None):
return X
class TagsBinarizer(BaseEstimator, TransformerMixin):
def __init__(self, part=0.01):
self.part = part
self.ts = TagsSimplifier(part=self.part)
self.mlb = MultiLabelBinarizer()
def fit(self, X, y=None):
simp_X = self.ts.fit_transform(X)
self.mlb.fit(simp_X)
return self
def transform(self, X, y=None):
simp_X = self.ts.transform(X)
return self.mlb.transform(simp_X)
def inverse_transform(self, X, y=None):
return self.mlb.inverse_transform(X) |