Kaggle competition: Tabular Playground Series - Jan 2021
<SOS> metric: RMSE Kaggle data source: tabular-playground-series-jan-2021<categorify>
warnings.filterwarnings("ignore" )
apps_all = get_apps_all_encoded(apps_all)
 apps_all_train, apps_all_test = get_apps_all_train_test(apps_all )<load_pretrained>
train = pd.read_csv('.. /input/tabular-playground-series-jan-2021/train.csv')
 test = pd.read_csv('.. /input/tabular-playground-series-jan-2021/test.csv')
 sub = pd.read_csv('.. /input/tabular-playground-series-jan-2021/sample_submission.csv' )
clf = train_apps_all(apps_all_train )<save_to_csv>
features = ['cont1', 'cont2', 'cont3', 'cont4', 'cont5', 'cont6', 'cont7',
'cont8', 'cont9', 'cont10', 'cont11', 'cont12', 'cont13', 'cont14']
preds = clf.predict_proba(apps_all_test.drop(['SK_ID_CURR'], axis=1)) [:, 1 ]
apps_all_test['TARGET'] = preds
apps_all_test[['SK_ID_CURR', 'TARGET']].to_csv('bureau_baseline_04.csv', index=False )<set_options>
X = train_01[features]
y = train_01['target']
tqdm().pandas()
pd_ctx = pd.option_context('display.max_colwidth', 100)
 pd.set_option('display.float_format', lambda x: '%.3f' % x )<load_from_csv>
params_xgb = {'lambda': 1,
'alpha': 0,
'colsample_bytree': 1,
'subsample': 1,
'learning_rate': 0.05,
'max_depth': 6,
'min_child_weight': 3,
'random_state': 48}
TRAIN_FILE = '/kaggle/input/quora-insincere-questions-classification/train.csv'
TEST_FILE = '/kaggle/input/quora-insincere-questions-classification/test.csv'
df = pd.read_csv(TRAIN_FILE)
 df.info()
test_df = pd.read_csv(TEST_FILE)
 with pd_ctx:
print("Sincere question")
 display(df[df['target'] == 0].head())
 print("Insincere question")
 display(df[df['target'] == 1].head() )<feature_engineering>
model_xgb = xgb.XGBRegressor(**params_xgb )
def statistic(df):
stats = pd.DataFrame() ;
stats['question_text'] = df['question_text']
stats['sp_char_words'] = stats['question_text'].str.findall(r'[^a-zA-Z0-9 ]' ).str.len()
stats['num_capital'] = stats['question_text'].progress_map(lambda x: len([c for c in str(x)if c.isupper() ]))
stats['num_numerics'] = stats['question_text'].progress_map(lambda x: sum(c.isdigit() for c in x))
stats['num_stopwords'] = stats['question_text'].progress_map(lambda x: len([c for c in str(x ).lower().split() if c in STOPWORDS]))
return stats 
<define_variables>
param_range = np.linspace(0, 1, 10)
 param_range
contractions= {"i'm": 'i am',"i'm'a": 'i am about to',"i'm'o": 'i am going to',"i've": 'i have',"i'll": 'i will',"i'll've": 'i will have',"i'd": 'i would',"i'd've": 'i would have',"Whatcha": 'What are you',"amn't": 'am not',"ain't": 'are not',"aren't": 'are not',"'cause": 'because',"can't": 'can not',"can't've": 'can not have',"could've": 'could have',"couldn't": 'could not',"couldn't've": 'could not have',"daren't": 'dare not',"daresn't": 'dare not',"dasn't": 'dare not',"didn't": 'did not','didn’t': 'did not',"don't": 'do not','don’t': 'do not',"doesn't": 'does not',"e'er": 'ever',"everyone's": 'everyone is',"finna": 'fixing to',"gimme": 'give me',"gon't": 'go not',"gonna": 'going to',"gotta": 'got to',"hadn't": 'had not',"hadn't've": 'had not have',"hasn't": 'has not',"haven't": 'have not',"he've": 'he have',"he's": 'he is',"he'll": 'he will',"he'll've": 'he will have',"he'd": 'he would',"he'd've": 'he would have',"here's": 'here is',"how're": 'how are',"how'd": 'how did',"how'd'y": 'how do you',"how's": 'how is',"how'll": 'how will',"isn't": 'is not',"it's": 'it is',"'tis": 'it is',"'twas": 'it was',"it'll": 'it will',"it'll've": 'it will have',"it'd": 'it would',"it'd've": 'it would have',"kinda": 'kind of',"let's": 'let us',"luv": 'love',"ma'am": 'madam',"may've": 'may have',"mayn't": 'may not',"might've": 'might have',"mightn't": 'might not',"mightn't've": 'might not have',"must've": 'must have',"mustn't": 'must not',"mustn't've": 'must not have',"needn't": 'need not',"needn't've": 'need not have',"ne'er": 'never',"o'": 'of',"o'clock": 'of the clock',"ol'": 'old',"oughtn't": 'ought not',"oughtn't've": 'ought not have',"o'er": 'over',"shan't": 'shall not',"sha'n't": 'shall not',"shalln't": 'shall not',"shan't've": 'shall not have',"she's": 'she is',"she'll": 'she will',"she'd": 'she would',"she'd've": 'she would have',"should've": 'should have',"shouldn't": 'should not',"shouldn't've": 'should not have',"so've": 'so have',"so's": 'so is',"somebody's": 'somebody is',"someone's": 'someone is',"something's": 'something is',"sux": 'sucks',"that're": 'that are',"that's": 'that is',"that'll": 'that will',"that'd": 'that would',"that'd've": 'that would have',"em": 'them',"there're": 'there are',"there's": 'there is',"there'll": 'there will',"there'd": 'there would',"there'd've": 'there would have',"these're": 'these are',"they're": 'they are',"they've": 'they have',"they'll": 'they will',"they'll've": 'they will have',"they'd": 'they would',"they'd've": 'they would have',"this's": 'this is',"those're": 'those are',"to've": 'to have',"wanna": 'want to',"wasn't": 'was not',"we're": 'we are',"we've": 'we have',"we'll": 'we will',"we'll've": 'we will have',"we'd": 'we would',"we'd've": 'we would have',"weren't": 'were not',"what're": 'what are',"what'd": 'what did',"what've": 'what have',"what's": 'what is',"what'll": 'what will',"what'll've": 'what will have',"when've": 'when have',"when's": 'when is',"where're": 'where are',"where'd": 'where did',"where've": 'where have',"where's": 'where is',"which's": 'which is',"who're": 'who are',"who've": 'who have',"who's": 'who is',"who'll": 'who will',"who'll've": 'who will have',"who'd": 'who would',"who'd've": 'who would have',"why're": 'why are',"why'd": 'why did',"why've": 'why have',"why's": 'why is',"will've": 'will have',"won't": 'will not',"won't've": 'will not have',"would've": 'would have',"wouldn't": 'would not',"wouldn't've": 'would not have',"y'all": 'you all',"y'all're": 'you all are',"y'all've": 'you all have',"y'all'd": 'you all would',"y'all'd've": 'you all would have',"you're": 'you are',"you've": 'you have',"you'll've": 'you shall have',"you'll": 'you will',"you'd": 'you would',"you'd've": 'you would have','jan.': 'january','feb.': 'february','mar.': 'march','apr.': 'april','jun.': 'june','jul.': 'july','aug.': 'august','sep.': 'september','oct.': 'october','nov.': 'november','dec.': 'december','I’m': 'I am','I’m’a': 'I am about to','I’m’o': 'I am going to','I’ve': 'I have','I’ll': 'I will','I’ll’ve': 'I will have','I’d': 'I would','I’d’ve': 'I would have','amn’t': 'am not','ain’t': 'are not','aren’t': 'are not','’cause': 'because','can’t': 'can not','can’t’ve': 'can not have','could’ve': 'could have','couldn’t': 'could not','couldn’t’ve': 'could not have','daren’t': 'dare not','daresn’t': 'dare not','dasn’t': 'dare not','doesn’t': 'does not','e’er': 'ever','everyone’s': 'everyone is','gon’t': 'go not','hadn’t': 'had not','hadn’t’ve': 'had not have','hasn’t': 'has not','haven’t': 'have not','he’ve': 'he have','he’s': 'he is','he’ll': 'he will','he’ll’ve': 'he will have','he’d': 'he would','he’d’ve': 'he would have','here’s': 'here is','how’re': 'how are','how’d': 'how did','how’d’y': 'how do you','how’s': 'how is','how’ll': 'how will','isn’t': 'is not','it’s': 'it is','’tis': 'it is','’twas': 'it was','it’ll': 'it will','it’ll’ve': 'it will have','it’d': 'it would','it’d’ve': 'it would have','let’s': 'let us','ma’am': 'madam','may’ve': 'may have','mayn’t': 'may not','might’ve': 'might have','mightn’t': 'might not','mightn’t’ve': 'might not have','must’ve': 'must have','mustn’t': 'must not','mustn’t’ve': 'must not have','needn’t': 'need not','needn’t’ve': 'need not have','ne’er': 'never','o’': 'of','o’clock': 'of the clock','ol’': 'old','oughtn’t': 'ought not','oughtn’t’ve': 'ought not have','o’er': 'over','shan’t': 'shall not','sha’n’t': 'shall not','shalln’t': 'shall not','shan’t’ve': 'shall not have','she’s': 'she is','she’ll': 'she will','she’d': 'she would','she’d’ve': 'she would have','should’ve': 'should have','shouldn’t': 'should not','shouldn’t’ve': 'should not have','so’ve': 'so have','so’s': 'so is','somebody’s': 'somebody is','someone’s': 'someone is','something’s': 'something is','that’re': 'that are','that’s': 'that is','that’ll': 'that will','that’d': 'that would','that’d’ve': 'that would have','there’re': 'there are','there’s': 'there is','there’ll': 'there will','there’d': 'there would','there’d’ve': 'there would have','these’re': 'these are','they’re': 'they are','they’ve': 'they have','they’ll': 'they will','they’ll’ve': 'they will have','they’d': 'they would','they’d’ve': 'they would have','this’s': 'this is','those’re': 'those are','to’ve': 'to have','wasn’t': 'was not','we’re': 'we are','we’ve': 'we have','we’ll': 'we will','we’ll’ve': 'we will have','we’d': 'we would','we’d’ve': 'we would have','weren’t': 'were not','what’re': 'what are','what’d': 'what did','what’ve': 'what have','what’s': 'what is','what’ll': 'what will','what’ll’ve': 'what will have','when’ve': 'when have','when’s': 'when is','where’re': 'where are','where’d': 'where did','where’ve': 'where have','where’s': 'where is','which’s': 'which is','who’re': 'who are','who’ve': 'who have','who’s': 'who is','who’ll': 'who will','who’ll’ve': 'who will have','who’d': 'who would','who’d’ve': 'who would have','why’re': 'why are','why’d': 'why did','why’ve': 'why have','why’s': 'why is','will’ve': 'will have','won’t': 'will not','won’t’ve': 'will not have','would’ve': 'would have','wouldn’t': 'would not','wouldn’t’ve': 'would not have','y’all': 'you all','y’all’re': 'you all are','y’all’ve': 'you all have','y’all’d': 'you all would','y’all’d’ve': 'you all would have','you’re': 'you are','you’ve': 'you have','you’ll’ve': 'you shall have','you’ll': 'you will','you’d': 'you would','you’d’ve': 'you would have'}
missing_spell = {'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', "mastrubating": 'masturbating', 'pennis': 'penis', 'Etherium': 'bitcoin', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', "whst": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization','electroneum':'bitcoin','nanodegree':'degree','hotstar':'star','dream11':'dream','ftre':'fire','tensorflow':'framework','unocoin':'bitcoin','lnmiit':'limit','unacademy':'academy','altcoin':'bitcoin','altcoins':'bitcoin','litecoin':'bitcoin','coinbase':'bitcoin','cryptocurency':'cryptocurrency','simpliv':'simple','quoras':'quora','schizoids':'psychopath','remainers':'remainder','twinflame':'soulmate','quorans':'quora','brexit':'demonetized','iiest':'institute','dceu':'comics','pessat':'exam','uceed':'college','bhakts':'devotee','boruto':'anime','cryptocoin':'bitcoin','blockchains':'blockchain','fiancee':'fiance','redmi':'smartphone','oneplus':'smartphone','qoura':'quora','deepmind':'framework','ryzen':'cpu','whattsapp':'whatsapp','undertale':'adventure','zenfone':'smartphone','cryptocurencies':'cryptocurrencies','koinex':'bitcoin','zebpay':'bitcoin','binance':'bitcoin','whtsapp':'whatsapp','reactjs':'framework','bittrex':'bitcoin','bitconnect':'bitcoin','bitfinex':'bitcoin','yourquote':'your quote','whyis':'why is','jiophone':'smartphone','dogecoin':'bitcoin','onecoin':'bitcoin','poloniex':'bitcoin','7700k':'cpu','angular2':'framework','segwit2x':'bitcoin','hashflare':'bitcoin','940mx':'gpu','openai':'framework','hashflare':'bitcoin','1050ti':'gpu','nearbuy':'near buy','freebitco':'bitcoin','antminer':'bitcoin','filecoin':'bitcoin','whatapp':'whatsapp','empowr':'empower','1080ti':'gpu','crytocurrency':'cryptocurrency','8700k':'cpu','whatsaap':'whatsapp','g4560':'cpu','payymoney':'pay money','fuckboys':'fuck boys','intenship':'internship','zcash':'bitcoin','demonatisation':'demonetization','narcicist':'narcissist','mastuburation':'masturbation','trignometric':'trigonometric','cryptocurreny':'cryptocurrency','howdid':'how did','crytocurrencies':'cryptocurrencies','phycopath':'psychopath','bytecoin':'bitcoin','possesiveness':'possessiveness','scollege':'college','humanties':'humanities','altacoin':'bitcoin','demonitised':'demonetized','brasília':'brazilia','accolite':'accolyte','econimics':'economics','varrier':'warrier','quroa':'quora','statergy':'strategy','langague':'language','splatoon':'game','7600k':'cpu','gate2018':'gate 2018','in2018':'in 2018','narcassist':'narcissist','jiocoin':'bitcoin','hnlu':'hulu','7300hq':'cpu','weatern':'western','interledger':'blockchain','deplation':'deflation', 'cryptocurrencies':'cryptocurrency', 'bitcoin':'blockchain cryptocurrency'}
def clean_tag(x):
if '[math]' in x:
x = re.sub('\[math\].*?math\]', 'math equation', x)
 if 'http' in x or 'www' in x:
x = re.sub('(?:(?:https?|ftp):\/\/)?[\w/\-?=%.]+\.[\w/\-?=%.]+', 'url', x)
 return x
def contraction_fix(word):
try:
a=contractions[word]
except KeyError:
a=word
return a
def misspell_fix(word):
try:
a=missing_spell[word]
except KeyError:
a=word
return a
def clean_text(text):
text = clean_tag(text)
 text = " ".join([contraction_fix(w)for w in text.split() ])
 text = " ".join([misspell_fix(w)for w in text.split() ])
 text = re.sub(r'[^a-zA-Z0-9]', ' ', text)
 return text
def apply_clean_text(question_text):
tmp = pd.DataFrame()
tmp['question_text'] = question_text;
tmp['clean'] = tmp.question_text.progress_map(clean_text)
 with pd_ctx:
display(tmp)
 return tmp['clean']
sample['clean'] = apply_clean_text(sample.question_text )<string_transform>
param_name = "alpha"
OOV_TOKEN = '<OOV>'
def create_tokenizer(docs):
tokenizer = Tokenizer(oov_token=OOV_TOKEN)
 tokenizer.fit_on_texts(list(docs))
print("Size of vocabulary: ", len(tokenizer.word_index))
return tokenizer
tokenizer = create_tokenizer(sample['clean'])
 print("20 từ đầu tiên trong từ điển:")
 list(tokenizer.word_index.items())[:20]<string_transform>
param_name = "lambda"
word_sequences = tokenizer.texts_to_sequences(sample['clean'])
 print("Length of 20 first word_sequences:")
 print(list(map(lambda x: len(x),word_sequences[:20])))
 print("
20 first word_sequences:")
 for sequence in word_sequences[:20]:
print(sequence )<categorify>
param_range = np.linspace(0.1, 1, 10)
 param_range
MAX_SENTENCE_LENGTH = 60
PADDING_TYPE = 'post'
TRUNCATE_TYPE = 'post'
def create_sequence(tokenizer, docs):
word_sequeces = tokenizer.texts_to_sequences(docs)
 padded_word_sequences = pad_sequences(word_sequeces, maxlen=MAX_SENTENCE_LENGTH, padding=PADDING_TYPE, truncating=TRUNCATE_TYPE)
 return padded_word_sequences
padded_sequences = create_sequence(tokenizer, sample['clean'])
 print("Kích thước mảng:",padded_sequences.shape)
 print("Length of 20 first word_sequences:")
 print(list(map(lambda x: len(x),padded_sequences[:20])))
 print("
10 first word_sequences:")
 for sequence in padded_sequences[:10]:
print(sequence)
<prepare_x_and_y>
param_name = 'colsample_bytree'
trainY = train.target
print("Clean train question")
 trainX_text = apply_clean_text(train.question_text)
 print("Clean test question")
 testX_text = apply_clean_text(test_df.question_text )<split>
param_name = 'subsample'
X_train, X_val, y_train, y_val = train_test_split(trainX_text, trainY, test_size=0.2, random_state=123 )<choose_model_class>
param_name = 'n_estimators'
EMBEDDING_DIM = 300
learning_rate = 0.001
def createModel_bidirectional_LSTM_GRU(features,embedding_matrix = None):
output_bias = Constant(np.log([len(data_pos)/len(data_neg)]))
x_input = Input(shape=(MAX_SENTENCE_LENGTH))
if not(embedding_matrix is None):
embedding = Embedding(features, EMBEDDING_DIM, input_length=MAX_SENTENCE_LENGTH, weights=[embedding_matrix], trainable=False )(x_input)
 else:
embedding = Embedding(features, EMBEDDING_DIM, input_length=MAX_SENTENCE_LENGTH )(x_input)
 x = SpatialDropout1D(0.2 )(embedding)
 lstm = Bidirectional(LSTM(256, return_sequences=True))(x)
 gru = Bidirectional(GRU(128, return_sequences=True))(lstm)
 x = Concatenate()([lstm, gru])
 x = GlobalAveragePooling1D()(x)
 x_output = Dense(1, activation='sigmoid', bias_initializer=output_bias )(x)
 model = Model(inputs=x_input, outputs=x_output)
 opt = Adam(lr=learning_rate)
 model.compile(loss='binary_crossentropy', optimizer= opt, metrics=[f1_m])
 return model
def f1_m(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
 recall = true_positives /(possible_positives + K.epsilon())
 precision = true_positives /(predicted_positives + K.epsilon())
 return 2*(( precision*recall)/(precision+recall+K.epsilon()))<compute_test_metric>
param_range = [100, 200, 300, 400, 500, 600, 700, 800, 900, 1000]
def best_threshold(y_train,train_preds):
tmp = [0,0,0]
delta = 0
for tmp[0] in tqdm(np.arange(0.1, 0.9, 0.01)) :
tmp[1] = metrics.f1_score(y_train, np.array(train_preds)>tmp[0])
 if tmp[1] > tmp[2]:
delta = tmp[0]
tmp[2] = tmp[1]
return delta, tmp[2]<choose_model_class>
X = train[features]
y = train['target']
strategy = None
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect()
strategy = tf.distribute.TPUStrategy(tpu)
 print('Use TPU')
 except ValueError:
if len(tf.config.list_physical_devices('GPU')) > 0:
strategy = tf.distribute.MirroredStrategy()
print('Use GPU')
 else:
strategy = tf.distribute.get_strategy()
print('Use CPU' )<choose_model_class>
def objective(trial,data=X,target=y):
train_x, test_x, train_y, test_y = train_test_split(data, target, test_size=0.15,random_state=42)
 param = {
'tree_method':'gpu_hist',
'lambda': trial.suggest_loguniform('lambda', 1e-3, 1),
'alpha': trial.suggest_loguniform('alpha', 1e-3, 1),
'colsample_bytree': trial.suggest_categorical('colsample_bytree', [0.1, 0.2, 0.3,0.5,0.7,0.9]),
'subsample': trial.suggest_categorical('subsample', [0.1, 0.2,0.3,0.4,0.5,0.8,1.0]),
'learning_rate': trial.suggest_categorical('learning_rate', [0.0008, 0.01, 0.015, 0.02,0.03, 0.05,0.08,0.1]),
'n_estimators': 4000,
'max_depth': trial.suggest_categorical('max_depth', [5,7,9,11,13,15,17,20,23,25]),
'random_state': 48,
'min_child_weight': trial.suggest_int('min_child_weight', 1, 400),
}
model = xgb.XGBRegressor(**param)
 model.fit(train_x,train_y,eval_set=[(test_x,test_y)],early_stopping_rounds=100,verbose=False)
 preds = model.predict(test_x)
 rmse = mean_squared_error(test_y, preds,squared=False)
 return rmse
class_weight = {
0: 1,
1: 3,
}
batch_size = 1024
n_epochs = 30
early_stopping=tf.keras.callbacks.EarlyStopping(
 monitor="val_loss",
patience=3,
mode="min",
restore_best_weights=True 
)
 reduce_lr=tf.keras.callbacks.ReduceLROnPlateau(
 monitor="val_loss",
factor=0.2,
patience=2,
verbose=1,
mode="auto" 
)
 my_callbacks=[early_stopping,reduce_lr]
def train_model_and_predict(X_train, y_train, X_val, y_val, X_test, embedding_vec=None):
print("Create vocab...")
 tokenizer = create_tokenizer(X_train)
 print("Create sequences...")
 X_train_seq = create_sequence(tokenizer, X_train)
 X_val_seq = create_sequence(tokenizer, X_val)
 X_test_seq = create_sequence(tokenizer, X_test)
 embedding_matrix = None
if not(embedding_vec is None):
embedding_matrix = load_embedding(embedding_vec, tokenizer.word_index)
 model = createModel_bidirectional_LSTM_GRU(len(tokenizer.word_index)+1, embedding_matrix)
 model.fit(X_train_seq, y_train, batch_size=batch_size, epochs=n_epochs, validation_data=(X_val_seq, y_val), class_weight=class_weight, callbacks=my_callbacks)
 val_pred = model.predict(X_val_seq, verbose=1, batch_size=256)
 threshold, f1_score = best_threshold(y_val, val_pred)
 test_pred = model.predict(X_test_seq, verbose=1, batch_size=256)
 return threshold, f1_score, val_pred, test_pred<predict_on_test>
study = optuna.create_study(direction='minimize')
 study.optimize(objective, n_trials=50)
 print('Number of finished trials:', len(study.trials))
print('Best trial:', study.best_trial.params )
with strategy.scope() :
threshold, f1_score, val_pred, test_pred = train_model_and_predict(X_train, y_train, X_val, y_val, testX_text)
 print(metrics.classification_report(y_val,(val_pred>threshold ).astype(int)) )<feature_engineering>
study.best_trial.params
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 def get_lines_count(file_name):
return sum(1 for _ in open(file_name, encoding="utf8", errors='ignore'))
def load_vec(file_name):
return dict(
 get_coefs(*o.split(" "))
for o in tqdm(open(
 file_name, encoding="utf8", errors='ignore'),
total=get_lines_count(file_name)
)if len(o)> 100
)<choose_model_class>
Best_params_xgb = {'lambda': 0.0014311714230223992,
'alpha': 0.008850567457271379,
'colsample_bytree': 0.3,
'subsample': 1.0,
'learning_rate': 0.01,
'max_depth': 20,
'min_child_weight': 245,
'n_estimators': 4000,
'random_state': 48,
'tree_method':'gpu_hist'}
EMBEDDING_DIM = 300
ps = PorterStemmer()
lc = LancasterStemmer()
sb = SnowballStemmer('english')
 def load_embedding(word2vec, word2index):
oov_count = 0
vocab_count = 0
vocab_size = len(word2index)
 embedding_weights = np.zeros(( vocab_size+1, EMBEDDING_DIM))
unknown_vector = np.zeros(( EMBEDDING_DIM,), dtype=np.float32)- 1
unknown_words = []
for key, i in tqdm(word2index.items()):
word = key
if word in word2vec:
embedding_weights[i] = word2vec[word]
continue
word = key.capitalize()
if word in word2vec:
embedding_weights[i] = word2vec[word]
continue
word = key.upper()
if word in word2vec:
embedding_weights[i] = word2vec[word]
continue
word = key.lower()
if word in word2vec:
embedding_weights[i] = word2vec[word]
continue
word = ps.stem(key)
 if word in word2vec:
embedding_weights[i] = word2vec[word]
continue
word = lc.stem(key)
 if word in word2vec:
embedding_weights[i] = word2vec[word]
continue
word = sb.stem(key)
 if word in word2vec:
embedding_weights[i] = word2vec[word]
continue
unknown_words.append(key)
 embedding_weights[i] = unknown_vector
print('Top 10 Null word embeddings: ')
 print(unknown_words[:10])
 print('
Null word embeddings: %d' % len(unknown_words))
print("There are {:.2f}% word out of embedding file".format(100*len(unknown_words)/vocab_size))
return embedding_weights<load_pretrained>
train_x, test_x, train_y, test_y = train_test_split(X, y, test_size=0.15,random_state=42)
 model_xgb = xgb.XGBRegressor(**Best_params_xgb)
 model_xgb.fit(train_x,train_y,eval_set=[(test_x,test_y)],early_stopping_rounds=100,verbose=False )
GLOVE_FILE = 'glove.840B.300d/glove.840B.300d.txt'
!unzip -n /kaggle/input/quora-insincere-questions-classification/embeddings.zip {GLOVE_FILE} -d.
 print('loading glove_vec')
 glove_vec = load_vec(GLOVE_FILE )<predict_on_test>
preds = model_xgb.predict(test_x)
 rmse = mean_squared_error(test_y, preds,squared=False)
 rmse
with strategy.scope() :
glove_threshold, glove_f1_score, glove_val_pred, glove_test_pred = train_model_and_predict(X_train, y_train, X_val, y_val, testX_text, glove_vec)
 print(metrics.classification_report(y_val,(glove_val_pred>glove_threshold ).astype(int)) )<set_options>
test_X = test[features]
del glove_vec
gc.collect()<load_pretrained>
preds = model_xgb.predict(test_X )
PARA_FILE = 'paragram_300_sl999/paragram_300_sl999.txt'
!unzip -n /kaggle/input/quora-insincere-questions-classification/embeddings.zip {PARA_FILE} -d.
 print('loading para_vec')
 para_vec = load_vec(PARA_FILE )<predict_on_test>
sub['target']=preds
sub.to_csv('submission.csv', index=False )
with strategy.scope() :
para_threshold, para_f1_score, para_val_pred, para_test_pred = train_model_and_predict(X_train, y_train, X_val, y_val, testX_text, para_vec)
 print(metrics.classification_report(y_val,(para_val_pred>para_threshold ).astype(int)) )<set_options>
X = train_01[features]
y = train_01['target']
del para_vec
gc.collect()<load_pretrained>
params_lgb = {'num_leaves': 31,
'min_data_in_leaf': 20,
'min_child_weight': 0.001,
'max_depth': -1,
'learning_rate': 0.005,
'bagging_fraction': 1,
'feature_fraction': 1,
'lambda_l1': 0,
'lambda_l2': 0,
'random_state': 48}
WIKI_FILE = 'wiki-news-300d-1M/wiki-news-300d-1M.vec'
!unzip -n /kaggle/input/quora-insincere-questions-classification/embeddings.zip {WIKI_FILE} -d.
 print('loading wiki_vec')
 wiki_vec = load_vec(WIKI_FILE )<predict_on_test>
model_lgb = lgb.LGBMRegressor(**params_lgb )
with strategy.scope() :
wiki_threshold, wiki_f1_score, wiki_val_pred, wiki_test_pred = train_model_and_predict(X_train, y_train, X_val, y_val, testX_text, wiki_vec)
 print(metrics.classification_report(y_val,(wiki_val_pred>wiki_threshold ).astype(int)) )<set_options>
param_range = np.linspace(0, 1, 10)
 param_range
del wiki_vec
gc.collect()<compute_test_metric>
param_name = 'lambda_l1'
val_prod = np.zeros(( len(X_val),), dtype=np.float32)
 val_prod += 1/3 * np.squeeze(glove_val_pred)
 val_prod += 1/3 * np.squeeze(para_val_pred)
 val_prod += 1/3 *np.squeeze(wiki_val_pred)
 threshold_global, f1_global = best_threshold(y_val, val_prod)
 print(metrics.classification_report(y_val,(val_prod>threshold_global ).astype(int)) )<save_to_csv>
param_name = 'lambda_l2'
pred_prob = np.zeros(( len(testX_text),), dtype=np.float32)
 pred_prob += 1/3 * np.squeeze(glove_test_pred)
 pred_prob += 1/3 * np.squeeze(para_test_pred)
 pred_prob += 1/3 * np.squeeze(wiki_test_pred)
 y_test_pre=(( pred_prob>threshold_global ).astype(int))
submit = pd.DataFrame()
submit["qid"]=test_df.qid
submit["prediction"]=y_test_pre
submit.to_csv("submission.csv",index=False )<import_modules>
param_name = 'feature_fraction'
import numpy as np
import pandas as pd
import os
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Dropout
import pandas as pd
import matplotlib.pyplot as plt<load_from_csv>
param_name = 'bagging_fraction'
train = np.loadtxt(open('/kaggle/input/digit-recognizer/train.csv', 'r'), delimiter=',', skiprows=1, dtype='float32')
 test = np.loadtxt(open('/kaggle/input/digit-recognizer/test.csv', 'r'), delimiter=',', skiprows=1, dtype='float32')
 train_images = train[:, 1:].reshape(( train.shape[0], 28, 28, 1)) / 255.0
train_labels = train[:, 0].astype(np.uint8)
 test_images = test.reshape(( test.shape[0], 28, 28, 1)) / 255.0<choose_model_class>
param_name = 'n_estimators'
augmentation_layer = tf.keras.Sequential([
tf.keras.layers.experimental.preprocessing.RandomRotation(0.1, input_shape=(28, 28, 1)) ,
tf.keras.layers.experimental.preprocessing.RandomZoom(( 0.2, 0.2)) ,
] )<categorify>
param_range = [100, 200, 300, 400, 500, 600, 700, 800, 900, 1000]
for i in range(5):
new_img = augmentation_layer(train_images[np.random.randint(train_images.shape[0])] ).numpy()
plt.imshow(new_img.reshape(( 28, 28)))
 plt.show()<choose_model_class>
X = train[features]
y = train['target']
model = Sequential([
tf.keras.layers.Input(( 28, 28, 1)) ,
augmentation_layer,
Conv2D(32, 3, activation='relu', padding="same"),
MaxPooling2D(2),
Conv2D(64, 3, activation='relu', padding="same"),
MaxPooling2D(2),
Conv2D(64, 3, activation='relu', padding="same"),
MaxPooling2D(2),
Flatten() ,
Dropout(0.3),
Dense(128, activation='relu'),
Dropout(0.3),
Dense(10, activation='softmax')
 ] )<choose_model_class>
def objective_lgb(trial,data=X,target=y):
train_x, test_x, train_y, test_y = train_test_split(data, target, test_size=0.15,random_state=42)
 param = {
'tree_method':'gpu_hist',
'lambda_l2': trial.suggest_loguniform('lambda_l2', 1e-3, 1),
'lambda_l1': trial.suggest_loguniform('lambda_l1', 1e-3, 1),
'feature_framcion': trial.suggest_categorical('feature_framcion', [0.1, 0.2, 0.3,0.5,0.7,0.9]),
'bagging_fraction': trial.suggest_categorical('bagging_framcion', [0.1, 0.2,0.3,0.4,0.5,0.8,1.0]),
'learning_rate': trial.suggest_categorical('learning_rate', [0.0008, 0.01, 0.015, 0.02,0.03, 0.05,0.08,0.1]),
'n_estimators': 4000,
'num_leaves': trial.suggest_categorical('num_leaves', [31,50,150,200,250,300,350]),
'max_depth': trial.suggest_categorical('max_depth', [5,7,9,11,13,15,17,20,23,25]),
'min_data_in_leaf': trial.suggest_categorical('min_data_in_leaf', [10,20,30]),
'min_child_weight': trial.suggest_categorical('min_child_weight', [0.001,0.005, 0.01, 0.05, 0.1,0.5]),
'random_state': 48
}
model = lgb.LGBMRegressor(**param)
 model.fit(train_x,train_y,eval_set=[(test_x,test_y)],early_stopping_rounds=100,verbose=False)
 preds = model.predict(test_x)
 rmse = mean_squared_error(test_y, preds,squared=False)
 return rmse
model.compile(
 optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy']
)<train_model>
study = optuna.create_study(direction='minimize')
 study.optimize(objective_lgb, n_trials=50)
 print('Number of finished trials:', len(study.trials))
print('Best trial:', study.best_trial.params )
history = model.fit(train_images, train_labels, epochs=100 )<choose_model_class>
study.best_trial.params
predition_model = tf.keras.Sequential()
for layer in model.layers:
if layer != augmentation_layer:
predition_model.add(layer)
 predition_model.compile(loss="sparse_categorical_crossentropy", optimizer="adam" )<predict_on_test>
Best_params_lgb = {'lambda_l2': 0.013616569506899653,
'lambda_l1': 0.006495842188985166,
'feature_framcion': 0.3,
'bagging_framcion': 0.3,
'learning_rate': 0.015,
'num_leaves': 200,
'max_depth': 25,
'min_data_in_leaf': 30,
'min_child_weight': 0.001,
'n_estimators': 3000,
'random_state': 48,
'tree_method':'gpu_hist'}
test_labels = np.argmax(predition_model.predict(test_images), axis=-1)
 print(test_labels.shape )<save_to_csv>
train_x, test_x, train_y, test_y = train_test_split(X, y, test_size=0.15,random_state=42)
 model_lgb = lgb.LGBMRegressor(**Best_params_lgb)
 model_lgb.fit(train_x,train_y,eval_set=[(test_x,test_y)],
early_stopping_rounds=100,verbose=False )
image_ids = np.arange(1, test_labels.shape[0]+1)
 result = np.concatenate(( image_ids.reshape(image_ids.shape[0], 1), test_labels.reshape(test_labels.shape[0], 1)) , axis=1)
 df = pd.DataFrame(result, columns=["ImageId", "Label"], dtype='int')
 df.to_csv("submission.csv", index=False )<set_options>
preds = model_lgb.predict(test_x)
 rmse = mean_squared_error(test_y, preds,squared=False)
 rmse
<load_from_csv><EOS> 
test_X = test[features]
preds = model_lgb.predict(test_X)
 sub['target']=preds
sub.to_csv('submission_lgb.csv', index=False )