Kaggle competition: Tabular Playground Series - Jan 2021
<SOS> metric: RMSE Kaggle data source: tabular-playground-series-jan-2021<set_options>
import optuna
!pip install GPUtil
def free_gpu_cache() :
print("Initial GPU Usage")
 gpu_usage()
torch.cuda.empty_cache()
cuda.select_device(0)
 cuda.close()
cuda.select_device(0)
 for obj in gc.get_objects() :
if torch.is_tensor(obj):
del obj
gc.collect()
print("GPU Usage after emptying the cache")
 gpu_usage()<load_from_csv>
import xgboost as xgb
from catboost import CatBoostRegressor
import numpy as np
import pandas as pd
from sklearn.model_selection import KFold
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import train_test_split
train = pd.read_csv(".. /input/nlp-getting-started/train.csv")
 test = pd.read_csv(".. /input/nlp-getting-started/test.csv")
 sub= pd.read_csv(".. /input/nlp-getting-started/sample_submission.csv" )<define_variables>
train = pd.read_csv('.. /input/tabular-playground-series-jan-2021/train.csv')
 test = pd.read_csv('.. /input/tabular-playground-series-jan-2021/test.csv')
 sub = pd.read_csv('.. /input/tabular-playground-series-jan-2021/sample_submission.csv' )
abbreviations = {
"$" : " dollar ",
"€" : " euro ",
"4ao" : "for adults only",
"a.m" : "before midday",
"a3" : "anytime anywhere anyplace",
"aamof" : "as a matter of fact",
"acct" : "account",
"adih" : "another day in hell",
"afaic" : "as far as i am concerned",
"afaict" : "as far as i can tell",
"afaik" : "as far as i know",
"afair" : "as far as i remember",
"afk" : "away from keyboard",
"app" : "application",
"approx" : "approximately",
"apps" : "applications",
"asap" : "as soon as possible",
"asl" : "age, sex, location",
"atk" : "at the keyboard",
"ave." : "avenue",
"aymm" : "are you my mother",
"ayor" : "at your own risk",
"b&b" : "bed and breakfast",
"b+b" : "bed and breakfast",
"b.c" : "before christ",
"b2b" : "business to business",
"b2c" : "business to customer",
"b4" : "before",
"b4n" : "bye for now",
"b@u" : "back at you",
"bae" : "before anyone else",
"bak" : "back at keyboard",
"bbbg" : "bye bye be good",
"bbc" : "british broadcasting corporation",
"bbias" : "be back in a second",
"bbl" : "be back later",
"bbs" : "be back soon",
"be4" : "before",
"bfn" : "bye for now",
"blvd" : "boulevard",
"bout" : "about",
"brb" : "be right back",
"bros" : "brothers",
"brt" : "be right there",
"bsaaw" : "big smile and a wink",
"btw" : "by the way",
"bwl" : "bursting with laughter",
"c/o" : "care of",
"cet" : "central european time",
"cf" : "compare",
"cia" : "central intelligence agency",
"csl" : "can not stop laughing",
"cu" : "see you",
"cul8r" : "see you later",
"cv" : "curriculum vitae",
"cwot" : "complete waste of time",
"cya" : "see you",
"cyt" : "see you tomorrow",
"dae" : "does anyone else",
"dbmib" : "do not bother me i am busy",
"diy" : "do it yourself",
"dm" : "direct message",
"dwh" : "during work hours",
"e123" : "easy as one two three",
"eet" : "eastern european time",
"eg" : "example",
"embm" : "early morning business meeting",
"encl" : "enclosed",
"encl." : "enclosed",
"etc" : "and so on",
"faq" : "frequently asked questions",
"fawc" : "for anyone who cares",
"fb" : "facebook",
"fc" : "fingers crossed",
"fig" : "figure",
"fimh" : "forever in my heart",
"ft." : "feet",
"ft" : "featuring",
"ftl" : "for the loss",
"ftw" : "for the win",
"fwiw" : "for what it is worth",
"fyi" : "for your information",
"g9" : "genius",
"gahoy" : "get a hold of yourself",
"gal" : "get a life",
"gcse" : "general certificate of secondary education",
"gfn" : "gone for now",
"gg" : "good game",
"gl" : "good luck",
"glhf" : "good luck have fun",
"gmt" : "greenwich mean time",
"gmta" : "great minds think alike",
"gn" : "good night",
"g.o.a.t" : "greatest of all time",
"goat" : "greatest of all time",
"goi" : "get over it",
"gps" : "global positioning system",
"gr8" : "great",
"gratz" : "congratulations",
"gyal" : "girl",
"h&c" : "hot and cold",
"hp" : "horsepower",
"hr" : "hour",
"hrh" : "his royal highness",
"ht" : "height",
"ibrb" : "i will be right back",
"ic" : "i see",
"icq" : "i seek you",
"icymi" : "in case you missed it",
"idc" : "i do not care",
"idgadf" : "i do not give a damn fuck",
"idgaf" : "i do not give a fuck",
"idk" : "i do not know",
"ie" : "that is",
"i.e" : "that is",
"ifyp" : "i feel your pain",
"IG" : "instagram",
"iirc" : "if i remember correctly",
"ilu" : "i love you",
"ily" : "i love you",
"imho" : "in my humble opinion",
"imo" : "in my opinion",
"imu" : "i miss you",
"iow" : "in other words",
"irl" : "in real life",
"j4f" : "just for fun",
"jic" : "just in case",
"jk" : "just kidding",
"jsyk" : "just so you know",
"l8r" : "later",
"lb" : "pound",
"lbs" : "pounds",
"ldr" : "long distance relationship",
"lmao" : "laugh my ass off",
"lmfao" : "laugh my fucking ass off",
"lol" : "laughing out loud",
"ltd" : "limited",
"ltns" : "long time no see",
"m8" : "mate",
"mf" : "motherfucker",
"mfs" : "motherfuckers",
"mfw" : "my face when",
"mofo" : "motherfucker",
"mph" : "miles per hour",
"mr" : "mister",
"mrw" : "my reaction when",
"ms" : "miss",
"mte" : "my thoughts exactly",
"nagi" : "not a good idea",
"nbc" : "national broadcasting company",
"nbd" : "not big deal",
"nfs" : "not for sale",
"ngl" : "not going to lie",
"nhs" : "national health service",
"nrn" : "no reply necessary",
"nsfl" : "not safe for life",
"nsfw" : "not safe for work",
"nth" : "nice to have",
"nvr" : "never",
"nyc" : "new york city",
"oc" : "original content",
"og" : "original",
"ohp" : "overhead projector",
"oic" : "oh i see",
"omdb" : "over my dead body",
"omg" : "oh my god",
"omw" : "on my way",
"p.a" : "per annum",
"p.m" : "after midday",
"pm" : "prime minister",
"poc" : "people of color",
"pov" : "point of view",
"pp" : "pages",
"ppl" : "people",
"prw" : "parents are watching",
"ps" : "postscript",
"pt" : "point",
"ptb" : "please text back",
"pto" : "please turn over",
"qpsa" : "what happens",
"ratchet" : "rude",
"rbtl" : "read between the lines",
"rlrt" : "real life retweet",
"rofl" : "rolling on the floor laughing",
"roflol" : "rolling on the floor laughing out loud",
"rotflmao" : "rolling on the floor laughing my ass off",
"rt" : "retweet",
"ruok" : "are you ok",
"sfw" : "safe for work",
"sk8" : "skate",
"smh" : "shake my head",
"sq" : "square",
"srsly" : "seriously",
"ssdd" : "same stuff different day",
"tbh" : "to be honest",
"tbs" : "tablespooful",
"tbsp" : "tablespooful",
"tfw" : "that feeling when",
"thks" : "thank you",
"tho" : "though",
"thx" : "thank you",
"tia" : "thanks in advance",
"til" : "today i learned",
"tl;dr" : "too long i did not read",
"tldr" : "too long i did not read",
"tmb" : "tweet me back",
"tntl" : "trying not to laugh",
"ttyl" : "talk to you later",
"u" : "you",
"u2" : "you too",
"u4e" : "yours for ever",
"utc" : "coordinated universal time",
"w/" : "with",
"w/o" : "without",
"w8" : "wait",
"wassup" : "what is up",
"wb" : "welcome back",
"wtf" : "what the fuck",
"wtg" : "way to go",
"wtpa" : "where the party at",
"wuf" : "where are you from",
"wuzup" : "what is up",
"wywh" : "wish you were here",
"yd" : "yard",
"ygtr" : "you got that right",
"ynk" : "you never know",
"zzz" : "sleeping bored and tired"
}<string_transform>
columns = [col for col in train.columns.to_list() if col not in ['id','target']]
def remove_URL(text):
url = re.compile(r'https?://\S+|www\.\S+')
 return url.sub(r'URL',text)
 def remove_HTML(text):
html=re.compile(r'<.*?>')
 return html.sub(r'',text)
 def remove_not_ASCII(text):
text = ''.join([word for word in text if word in string.printable])
 return text
def word_abbrev(word):
return abbreviations[word.lower() ] if word.lower() in abbreviations.keys() else word
def replace_abbrev(text):
string = ""
for word in text.split() :
string += word_abbrev(word)+ " "
return string
def remove_mention(text):
at=re.compile(r'@\S+')
 return at.sub(r'USER',text)
 def remove_number(text):
num = re.compile(r'[-+]?[.\d]*[\d]+[:,.\d]*')
 return num.sub(r'NUMBER', text)
 def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'EMOJI', text)
 def transcription_sad(text):
eyes = "[8:=;]"
nose = "['`\-]"
smiley = re.compile(r'[8:=;]['\-]?[(\\/]')
 return smiley.sub(r'SADFACE', text)
 def transcription_smile(text):
eyes = "[8:=;]"
nose = "['`\-]"
smiley = re.compile(r'[8:=;]['\-]?[)dDp]')
 return smiley.sub(r'SMILE', text)
 def transcription_heart(text):
heart = re.compile(r'<3')
 return heart.sub(r'HEART', text)
 def remove_elongated_words(text):
rep = re.compile(r'\b(\S*? )([a-z])\2{2,}\b')
 return rep.sub(r'\1\2 ELONG', text)
 def remove_repeat_punct(text):
rep = re.compile(r'([!?.]){2,}')
 return rep.sub(r'\1 REPEAT', text)
 def remove_all_punct(text):
table = str.maketrans('','',string.punctuation)
 return text.translate(table)
 def remove_punct(text):
punctuations = '@
for p in punctuations:
text = text.replace(p, f' {p} ')
 text = text.replace('...', '...')
 if '...' not in text:
text = text.replace('.. ', '...')
 return text
def remove_stopwords(text):
text = ' '.join([word for word in text.split() if word not in(stopwords)])
 return text<drop_column>
data=train[columns]
target=train['target']
def clean_tweet(text):
text = remove_URL(text)
 text = remove_HTML(text)
 text = remove_not_ASCII(text)
 text = text.lower()
text = replace_abbrev(text)
 text = remove_mention(text)
 text = remove_number(text)
 text = remove_emoji(text)
 text = transcription_sad(text)
 text = transcription_smile(text)
 text = transcription_heart(text)
 text = remove_elongated_words(text)
 text = remove_repeat_punct(text)
 return text<feature_engineering>
def objective(trial,data=data,target=target):
train_x, test_x, train_y, test_y = train_test_split(data, target, test_size=0.15,random_state=42)
 param = {
'tree_method':'gpu_hist',
'lambda': trial.suggest_loguniform('lambda', 1e-3, 10.0),
'alpha': trial.suggest_loguniform('alpha', 1e-3, 10.0),
'colsample_bytree': trial.suggest_categorical('colsample_bytree', [0.3,0.4,0.5,0.6,0.7,0.8,0.9, 1.0]),
'subsample': trial.suggest_categorical('subsample', [0.4,0.5,0.6,0.7,0.8,1.0]),
'learning_rate': trial.suggest_categorical('learning_rate', [0.008,0.009,0.01,0.012,0.014,0.016,0.018, 0.02]),
'n_estimators': 4000,
'max_depth': trial.suggest_categorical('max_depth', [5,7,9,11,13,15,17,20]),
'random_state': trial.suggest_categorical('random_state', [24, 48,2020]),
'min_child_weight': trial.suggest_int('min_child_weight', 1, 300),
}
model = xgb.XGBRegressor(**param)
 model.fit(train_x,train_y,eval_set=[(test_x,test_y)],early_stopping_rounds=100,verbose=False)
 preds = model.predict(test_x)
 rmse = mean_squared_error(test_y, preds,squared=False)
 return rmse
train["clean_text"] = train["text"].apply(clean_tweet)
 test["clean_text"] = test["text"].apply(clean_tweet)
 train["clean_tokens"] = train["clean_text"].apply(lambda x: word_tokenize(x))
test["clean_tokens"] = test["clean_text"].apply(lambda x: word_tokenize(x))<feature_engineering>
study = optuna.create_study(direction='minimize')
 study.optimize(objective, n_trials=50)
 print('Number of finished trials:', len(study.trials))
print('Best trial:', study.best_trial.params )
skip_gram_model = Word2Vec(train['clean_tokens'],size=150,window=3,min_count=2,sg=1)
 skip_gram_model.train(train['clean_tokens'],total_examples=len(train['clean_tokens']),epochs=10)
 cbow_model = Word2Vec(train['clean_tokens'],size=150,window=3,min_count=2)
 cbow_model.train(train['clean_tokens'],total_examples=len(train['clean_tokens']),epochs=10)
 train["clean_tokens"] = train["text"].apply(lambda x: word_tokenize(x))
test["clean_tokens"] = test["text"].apply(lambda x: word_tokenize(x))
def get_word_embeddings(token_list,vector,k=150):
if len(token_list)< 1:
return np.zeros(k)
 else:
vectorized = [vector[word] if word in vector else np.random.rand(k)for word in token_list]
sum = np.sum(vectorized,axis=0)
 return sum/len(vectorized )<feature_engineering>
study.trials_dataframe()
max_features=5000
count_vectorizer = CountVectorizer(max_features=max_features)
 sparce_matrix_train=count_vectorizer.fit_transform(train['clean_text'])
 sparce_matrix_test=count_vectorizer.fit_transform(train['clean_text'])
 def count_vector(data):
count_vectorizer = CountVectorizer()
vect = count_vectorizer.fit_transform(data)
 return vect, count_vectorizer
def tfidf_vector(data):
tfidf_vectorizer = TfidfVectorizer()
vect = tfidf_vectorizer.fit_transform(data)
 return vect, tfidf_vectorizer
def skip_gram_vectorizer(tokens,vector):
embeddings = tokens.apply(lambda x: get_word_embeddings(x, skip_gram_model))
return list(embeddings)
 def cbow_vectorizer(tokens,vector):
embeddings = tokens.apply(lambda x: get_word_embeddings(x, cbow_model))
return list(embeddings)
 X_train_count, count_vectorizer = count_vector(train['clean_text'])
 X_train_tfidf, tfidf_vectorizer = tfidf_vector(train['clean_text'])
 X_train_skip_gram = skip_gram_vectorizer(train['clean_tokens'],skip_gram_model)
 X_train_cbow = cbow_vectorizer(train['clean_tokens'],cbow_model)
 X_test_count = count_vectorizer.transform(test['clean_text'])
 X_test_tfidf = tfidf_vectorizer.transform(test['clean_text'])
 X_test_skip_gram = skip_gram_vectorizer(test['clean_tokens'],skip_gram_model)
 X_test_cbow = cbow_vectorizer(test['clean_tokens'],cbow_model)
 X_train_skip_gram = np.array(X_train_skip_gram)
 X_train_cbow = np.array(X_train_cbow)
 X_test_skip_gram = np.array(X_test_skip_gram)
 X_test_cbow = np.array(X_test_cbow)
 with open('count_vectorizer.pickle', 'wb')as handle:
pickle.dump(count_vectorizer, handle, protocol=pickle.HIGHEST_PROTOCOL)
 with open('tfidf_vectorizer.pickle', 'wb')as handle:
pickle.dump(tfidf_vectorizer, handle, protocol=pickle.HIGHEST_PROTOCOL)
 with open('skip_gram_model.pickle', 'wb')as handle:
pickle.dump(skip_gram_model, handle, protocol=pickle.HIGHEST_PROTOCOL)
 with open('cbow_model.pickle', 'wb')as handle:
pickle.dump(cbow_model, handle, protocol=pickle.HIGHEST_PROTOCOL)
 with open('cbow_model.pickle', 'wb')as handle:
pickle.dump(cbow_model, handle, protocol=pickle.HIGHEST_PROTOCOL)
 with open('cbow_model.pickle', 'wb')as handle:
pickle.dump(cbow_model, handle, protocol=pickle.HIGHEST_PROTOCOL)
<create_dataframe>
Best_trial= {'lambda': 0.0030282073258141168, 'alpha': 0.01563845128469084, 'colsample_bytree': 0.5,
'subsample': 0.7,'n_estimators': 4000, 'learning_rate': 0.01,'max_depth': 15,
'random_state': 2020, 'min_child_weight': 257,'tree_method':'gpu_hist'}
metrics = pd.DataFrame(columns=['model' ,'vectoriser', 'f1 score', 'train accuracy','test accuracy'] )<choose_model_class>
preds = np.zeros(test.shape[0])
 kf = KFold(n_splits=5,random_state=48,shuffle=True)
 rmse=[]
n=0
for trn_idx, test_idx in kf.split(train[columns],train['target']):
X_tr,X_val=train[columns].iloc[trn_idx],train[columns].iloc[test_idx]
y_tr,y_val=train['target'].iloc[trn_idx],train['target'].iloc[test_idx]
model = xgb.XGBRegressor(**Best_trial)
 model.fit(X_tr,y_tr,eval_set=[(X_val,y_val)],early_stopping_rounds=100,verbose=False)
 preds+=model.predict(test[columns])/kf.n_splits
rmse.append(mean_squared_error(y_val, model.predict(X_val), squared=False))
print(n+1,rmse[n])
 n+=1
models=[
XGBClassifier(max_depth=6, n_estimators=1000),
LogisticRegression(random_state=random_state),
SVC(random_state=random_state),
MultinomialNB() ,
DecisionTreeClassifier(random_state = random_state),
KNeighborsClassifier() ,
RandomForestClassifier(random_state=random_state),
]<split>
def objective(trial,data=data,target=target):
train_x, test_x, train_y, test_y = train_test_split(data, target, test_size=0.15,random_state=42)
 param = {
'loss_function': 'RMSE',
'task_type': 'GPU',
'l2_leaf_reg': trial.suggest_loguniform('l2_leaf_reg', 1e-3, 10.0),
'max_bin': trial.suggest_int('max_bin', 200, 400),
'subsample': trial.suggest_uniform('bagging_fraction', 0.4, 1.0),
'learning_rate': trial.suggest_uniform('learning_rate', 0.006, 0.018),
'n_estimators': 25000,
'max_depth': trial.suggest_categorical('max_depth', [7,10,14,16]),
'random_state': trial.suggest_categorical('random_state', [24, 48,2020]),
'min_data_in_leaf': trial.suggest_int('min_data_in_leaf', 1, 300),
}
model = CatBoostRegressor(**param)
 model.fit(train_x,train_y,eval_set=[(test_x,test_y)],early_stopping_rounds=200,verbose=False)
 preds = model.predict(test_x)
 rmse = mean_squared_error(test_y, preds,squared=False)
 return rmse
for model in models:
y = train.target
x = X_train_count
x_train, x_test, y_train, y_test = train_test_split(x,y, test_size = 0.3)
 fit_and_predict(model,x_train,x_test,y_train,y_test,'Count vector')
 x = X_train_tfidf
x_train, x_test, y_train, y_test = train_test_split(x,y, test_size = 0.3)
 fit_and_predict(model,x_train,x_test,y_train,y_test, 'Tfidf vector')
 x = X_train_skip_gram
x_train, x_test, y_train, y_test = train_test_split(x,y, test_size = 0.3)
 fit_and_predict(model,x_train,x_test,y_train,y_test, 'skip gram vector')
 x = X_train_cbow
x_train, x_test, y_train, y_test = train_test_split(x,y, test_size = 0.3)
 fit_and_predict(model,x_train,x_test,y_train,y_test, 'cbow vector' )<sort_values>
study = optuna.create_study(direction='minimize')
 study.optimize(objective, n_trials=50)
 print('Number of finished trials:', len(study.trials))
print('Best trial:', study.best_trial.params )
metrics = metrics.sort_values('f1 score',ascending=False )<find_best_params>
Best_trial = {'l2_leaf_reg': 0.02247766515106271, 'max_bin': 364, 'subsample': 0.6708650091202213,
'learning_rate': 0.010290546311954876, 'max_depth': 10, 'random_state': 24, 'min_data_in_leaf': 300,
'task_type': 'GPU','loss_function': 'RMSE','n_estimators': 25000}
free_gpu_cache()<import_modules>
preds = np.zeros(test.shape[0])
 kf = KFold(n_splits=10,random_state=48,shuffle=True)
 rmse=[]
n=0
for trn_idx, test_idx in kf.split(train[columns],train['target']):
X_tr,X_val=train[columns].iloc[trn_idx],train[columns].iloc[test_idx]
y_tr,y_val=train['target'].iloc[trn_idx],train['target'].iloc[test_idx]
model = CatBoostRegressor(**Best_trial)
 model.fit(X_tr,y_tr,eval_set=[(X_val,y_val)],early_stopping_rounds=200,verbose=False)
 preds+=model.predict(test[columns])/kf.n_splits
rmse.append(mean_squared_error(y_val, model.predict(X_val), squared=False))
print(n+1,rmse[n])
 n+=1
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow import keras
from keras.models import Sequential
from keras.layers import Dense, Embedding, LSTM,GRU, Dropout, Activation, Input, Flatten, Bidirectional, Conv1D, MaxPooling1D
from tensorflow.keras import layers
from keras import optimizers
from functools import reduce<train_on_grid>
Best_trial = {'l2_leaf_reg': 0.02247766515106271, 'max_bin': 364, 'subsample': 0.6708650091202213,
'learning_rate': 0.010290546311954876, 'max_depth': 10, 'random_state': 24, 'min_data_in_leaf': 300,
'loss_function': 'RMSE','n_estimators': 25000,'rsm':0.5}
def train_lstm(x_train,x_test,y_train,y_test,vectorizer_name,vocab_size,input_length):
epochs = 1
verbose = 1
batch_size = 32
embed_dim = 32
optimizer = optimizers.Adam(lr=0.002)
 model = Sequential()
model.add(Embedding(vocab_size, embed_dim,input_length = input_length))
model.add(Dropout(0.2))
model.add(LSTM(32, dropout=0.2, recurrent_dropout=0.4))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss = 'binary_crossentropy', optimizer=optimizer, metrics = ['accuracy'])
 print(model.summary())
 model.fit(x_train, y_train, epochs = epochs, verbose = verbose, batch_size=batch_size, validation_data=(x_test, y_test))
model_name = "LSTM using "+ vectorizer_name
model.save(model_name)
 y_pred = model.predict(x_test)
 f1score = f1_score(y_test,y_pred.round() ,average='weighted')
 train_accuracy = round(accuracy_score(y_train,reduce(list.__add__, model.predict(x_train ).round().tolist())) *100)
 test_accuracy = round(accuracy_score(y_test,y_pred.round())*100)
 global metrics
metrics = metrics.append({
'model': model_name,
'f1 score': f1score,
'train accuracy': train_accuracy,
'test accuracy': test_accuracy,
'vectoriser': vectorizer_name,
},
ignore_index=True 
)
 print(model_name)
 print(classification_report(y_test,y_pred.round()))
print('Accuracy of classifier on training set:{}%'.format(train_accuracy))
print('Accuracy of classifier on test set:{}%'.format(test_accuracy))<split>
preds = np.zeros(test.shape[0])
 kf = KFold(n_splits=10,random_state=48,shuffle=True)
 rmse=[]
n=0
for trn_idx, test_idx in kf.split(train[columns],train['target']):
X_tr,X_val=train[columns].iloc[trn_idx],train[columns].iloc[test_idx]
y_tr,y_val=train['target'].iloc[trn_idx],train['target'].iloc[test_idx]
model = CatBoostRegressor(**Best_trial)
 model.fit(X_tr,y_tr,eval_set=[(X_val,y_val)],early_stopping_rounds=100,verbose=False)
 preds+=model.predict(test[columns])/kf.n_splits
rmse.append(mean_squared_error(y_val, model.predict(X_val), squared=False))
print(n+1,rmse[n])
 n+=1
y = train['target'].values
x_train, x_test, y_train, y_test = train_test_split(X_train_skip_gram,y, test_size = 0.3)
 train_lstm(x_train,x_test,y_train,y_test, 'skip gram vector',5329,150)
<set_options>
sub['target']=preds
sub.to_csv('submission.csv', index=False )