Kaggle competition: SERPRO - Iris
<SOS> metric: categorizationaccuracy Kaggle data source: serpro-iris<string_transform>
import numpy as np
import pandas as pd
def share_unique(string_vec):

 share_list = []
for char in string_vec:
sh = len(set(char.split())) /len(char.split())
 share_list.append(sh)
 return share_list
def share_capslock(string_vec):

 share_list = []
for char in string_vec:
to_upper = char.upper().split()
sh = len(set(char.split() ).intersection(to_upper)) /len(char.split())
 share_list.append(sh)
 return share_list
def ends_with_symbol(string_vec, symbol):

 return [int(x.endswith(symbol)) for x in string_vec]
def count_words(string_vec):

 return [len(x.split())for x in string_vec]
def count_occurance(string_vec):

 return [x.count('!')+ x.count('?')for x in string_vec]<define_variables>
train_data = pd.read_csv('.. /input/iris-train.csv', index_col='Id')
 train_data.head()
embed_dict = {'google': {'path' : '.. /input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin',
'reading' : 'word2vec',
'binary': True},
'glove': {'path': '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt',
'reading' : 'text',
'binary': False},
'wiki': {'path' : '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec',
'reading' : 'text',
'binary': False},
'paragram': {'path': '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt',
'reading' : 'text',
'binary' : False}
}<define_variables>
test_data = pd.read_csv('.. /input/iris-test.csv', index_col='Id')
 test_data.head()
embeds_to_use = ['glove']<load_pretrained>
X_train = train_data.drop(['Species'], axis=1)
 y_train = train_data['Species']
print('Forma dos dados de treino:', X_train.shape, y_train.shape )
index = 0
for embed in embeds_to_use:
print('Reading: ' + embed)
 path = embed_dict.get(embed)['path']
type_of_file = embed_dict.get(embed)['reading']
bin = embed_dict.get(embed)['binary']
embedding = read_embedding(path, type_of_file, bin)
 if embed == embeds_to_use[0]:
model_embed = embedding
else:
all_keys = [k for k in embedding.keys() ]
index += 1
for key in tqdm(all_keys):
try:
vect = np.concatenate([model_embed[key], embedding[key]])
 except:
vect = np.concatenate([np.zeros(( 1, 300 * index), dtype = 'float32')[0], embedding[key]])
 model_embed.update({key : vect})
 del vect
del all_keys
del embedding
gc.collect()
time.sleep(5 )<define_variables>
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
vocab_mapper = {'Quorans' : 'Qurans',
'Blockchain' : 'blockchain',
'blockchains' : 'blockchain',
'demonetisation' : 'demonetization',
'ethereum' : 'Ethereum',
'Qoura' : 'Quora',
'SJWs' : 'SJW',
'bhakts' : 'bhakti',
'Bhakts' : 'Bhakti',
'kotlin' : 'Kotlin',
'narcissit' : 'narcissist',
'Trumpism' : 'Trump',
'Tamilans' : 'Tamilians',
'acturial' : 'actuarial',
'demonitization' : 'demonetization',
'Demonetization' : 'demonetization',
'demonitisation' : 'demonetization',
'Demonetisation' : 'demonetization',
'Whyis' : 'Why is',
'AirPods' : 'AirPod',
'Drumpf': 'Trumpf',
'Zhihu' : 'Zhihua',
'Neuralink' : 'Neurolink',
'fullform' : 'full-form',
'biharis' : 'Biharis',
'madheshi' : 'Madheshi',
'Xiomi' : 'Xiaomi',
'rohingya' : 'Rohingya',
'Despacito' : 'Desposito',
'schizoids' : 'schizoid',
'MHTCET' : 'MHT-CET',
'fortnite' : 'Fortnite',
'Bittrex' : 'Bitrex',
'ReactJS' : 'JavaScript',
'hyperloop' : 'Hyperloop',
'adhaar' : 'Aadhaar',
'Adhaar' : 'Aadhaar',
'Baahubali' : 'Bahubali',
'Cryptocurrency' : 'cryptocurrency',
'cryptocurrencies' : 'cryptocurrency',
'cryptocoins' : 'cryptocurrency',
"\u200b":" ",
"\ufeff" : "",
"2k17" : '2017',
"2k18" : '2018',
"nofap": 'no fap',
'Brexiting' : 'Brexit',
'mastuburation' : 'masturbation',
'quara' : 'Quora',
'Quoras' : 'Quora',
"fiancé" : "fiance",
'π' : 'pi',
'Pokémon' : 'Pokemon',
'€' : 'euro'
}
word_expansion = {"aren't" : 'are not',
"I'm" : 'I am',
"What's" : 'What is',
"don’t" : "do not",
"isn't" : "is not",
"I’m" : "I am",
'aren’t' : 'are not',
"Can't" : "cannot",
"can't" : "cannot",
"don't" : 'do not',
"How's" : "how is",
"we're" : 'we are',
"won't" : 'will not',
"they're" : "they are",
"he's" : "he is",
"doesn’t" : "does not",
"shouldn't" : "should not",
"Shouldn't" : "should not",
"hasn't" : "has not",
"couldn't" : "could not",
"I’ve" : "I have",
"aren't" : "are not",
"weren't" : 'were not'} 
<define_variables>
model = LogisticRegression()
model.fit(X_train, y_train)
 print(model )
punct = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√']
punct = ''.join(punct)
 def preproc_pipeline(string_df, model_embed, punct, word_expansion = None, vocab_mapper = None):

 if(word_expansion is not None):
string_df = spell_checker(string_df, word_expansion)
 string_df = clean_punct(string_df, punct)
 if(vocab_mapper is not None):
string_df = spell_checker(string_df, vocab_mapper)
 string_df = clean_digits(string_df)
 string_df = clean_ws(string_df)
 return string_df
X_tr = preproc_pipeline(X_train, model_embed, punct, vocab_mapper = vocab_mapper)
 X_te = preproc_pipeline(X_test, model_embed, punct, vocab_mapper = vocab_mapper )<feature_engineering>
y_pred = model.predict(X_test)
 print('Exemplos de previsões:
', y_pred[:10] )
vocab = build_vocab(X_tr)
 oov = check_coverage(vocab, model_embed)
 oov = [x[0] for x in oov if x[1] > 60]
synonyms = find_most_similar(oov, model_embed, return_top = 3)
 print(synonyms )<define_variables>
submission = pd.DataFrame({
'Id': X_test.index,
'Species': y_pred
})
 submission.set_index('Id', inplace=True)
 submission.head(10 )
<choose_model_class><EOS> 
submission.to_csv('iris-submission.csv' )