Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<train_model>
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
xgb_params = {'eta': 0.2,
'max_depth': 5,
'subsample': 0.8,
'colsample_bytree': 0.8,
'objective': 'binary:logistic',
'eval_metric': 'auc',
'seed': 1234
}
X_train, X_valid, y_train, y_valid = train_test_split(train_x, y, test_size=0.33, random_state=2018)
 d_train = xgb.DMatrix(X_train, y_train)
 d_valid = xgb.DMatrix(X_valid, y_valid)
 d_test = xgb.DMatrix(test_x )<train_model>
train_data = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 test_data = pd.read_csv('.. /input/nlp-getting-started/test.csv')
 sample_submission = pd.read_csv('.. /input/nlp-getting-started/sample_submission.csv')
 train_data
watchlist = [(d_train, 'train'),(d_valid, 'valid')]
model_xgb = xgb.train(xgb_params, d_train, 500, watchlist, verbose_eval=50, early_stopping_rounds=20 )<compute_train_metric>
count_data = train_data[['text','target']].groupby('target' ).count().reset_index()
count_data
xgb_pred_test = model_xgb.predict(d_test)
 xgb_pred_valid = model_xgb.predict(d_valid)
 auc = roc_auc_score(y_valid, xgb_pred_valid)
 print('AUC:',auc )<predict_on_test>
print(dataset.isnull().sum())
 print('-'*100)
 print(train_data.isnull().sum() )
xgb_pred_train = model_xgb.predict(d_train)
<load_pretrained>
dataset['len_letters'] = dataset['text'].apply(len)
 train_data['len_letters'] = train_data['text'].apply(len)
 dataset.head(5 )
eli5.show_weights(model_xgb,vec=vectorizer )<define_variables>
dataset['keyword'] = dataset['keyword'].fillna('not_known' )
final_preds=0.4*xgb_pred_test+0.6*test_preds<save_to_csv>
dataset['text'][dataset['keyword'] == 'blaze']
x_preds = pd.DataFrame(final_preds)
 x_preds.columns = ['project_is_approved']
sub_id=sample_sub['id']
submission = pd.concat([sub_id, x_preds], axis=1)
 submission.to_csv('lgbm_xgb_blend.csv', index=False )<train_model>
train_data['location'].isnull().sum()
end_time=time.time()
print("total time spent in FE and model",end_time-start_vect,"s")
 print("total time till end",end_time-start_time,"s" )<define_variables>
print('For disastrous, unknown locations : ',train_data['location'][train_data['target']==1].isnull().sum())
 print('For non disastrous, unknown locations : ',train_data['location'][train_data['target']==0].isnull().sum())
stop_words = stopwords.words('english')
 data_path = '.. /input/' 
<load_from_csv>
dataset['location'].fillna('unknown', inplace = True)
 train_data['location'].fillna('unknown', inplace = True )
train = pd.read_csv(data_path + 'donorschoose-application-screening/train.csv', dtype={"project_essay_3": object, "project_essay_4": object})
 target = train['project_is_approved']
train = train.drop('project_is_approved', axis=1)
 test = pd.read_csv(data_path + 'donorschoose-application-screening/test.csv', dtype={"project_essay_3": object, "project_essay_4": object})
 sub = pd.read_csv(data_path + 'donorschoose-application-screening/sample_submission.csv')
 resources = pd.read_csv(data_path + 'donorschoose-application-screening/resources.csv')
 train.fillna(( 'unk'), inplace=True)
 test.fillna(( 'unk'), inplace=True)
<drop_column>
num_locations = Counter(dataset['location'])
 num_locations
for i in range(2):
train['cat'+str(i)] = train['project_subject_categories'].str.split(',', expand=True)[i]
test['cat'+str(i)] = train['project_subject_categories'].str.split(',', expand=True)[i]
train['scat'+str(i)] = train['project_subject_subcategories'].str.split(',', expand=True)[i]
test['scat'+str(i)] = train['project_subject_subcategories'].str.split(',', expand=True)[i]
train.head()<categorify>
dataset['location'] = le().fit_transform(dataset['location'])
 dataset
df_all = pd.concat([train, test], axis=0)
 cols = [
'teacher_id',
'teacher_prefix',
'school_state',
'project_grade_category',
'project_subject_categories',
'project_subject_subcategories',
'cat0',
'scat0',
'cat1',
'scat1'
]
for c in tqdm(cols):
le = LabelEncoder()
le.fit(df_all[c].astype(str))
train[c] = le.transform(train[c].astype(str))
test[c] = le.transform(test[c].astype(str))
del df_all; gc.collect() 
<data_type_conversions>
dataset['text'] = dataset['text'].map(lambda x: x.lower())
 train_data['text'] = train_data['text'].map(lambda x: x.lower())
 dataset['text']
train['project_submitted_datetime'] = pd.to_datetime(train['project_submitted_datetime'])
 test['project_submitted_datetime'] = pd.to_datetime(test['project_submitted_datetime'])
 train['datetime_int'] = train['project_submitted_datetime'].apply(lambda x: x.strftime('%Y%m%d')).astype(int)
 test['datetime_int'] = test['project_submitted_datetime'].apply(lambda x: x.strftime('%Y%m%d')).astype(int)
 train['datetime_day'] = train['project_submitted_datetime'].dt.day
train['datetime_dow'] = train['project_submitted_datetime'].dt.dayofweek
train['datetime_year'] = train['project_submitted_datetime'].dt.year
train['datetime_month'] = train['project_submitted_datetime'].dt.month
train['datetime_hour'] = train['project_submitted_datetime'].dt.hour
train = train.drop('project_submitted_datetime', axis=1)
 test['datetime_day'] = test['project_submitted_datetime'].dt.day
test['datetime_dow'] = test['project_submitted_datetime'].dt.dayofweek
test['datetime_year'] = test['project_submitted_datetime'].dt.year
test['datetime_month'] = test['project_submitted_datetime'].dt.month
test['datetime_hour'] = test['project_submitted_datetime'].dt.hour
test = test.drop('project_submitted_datetime', axis=1)
 train['e1_length'] = train['project_essay_1'].apply(len)
 test['e1_length'] = train['project_essay_1'].apply(len)
 train['e2_length'] = train['project_essay_2'].apply(len)
 test['e2_length'] = train['project_essay_2'].apply(len)
 train['project_title_len'] = train['project_title'].apply(lambda x: len(str(x)))
 test['project_title_len'] = test['project_title'].apply(lambda x: len(str(x)))
 train['has_gt2_essays'] = train['project_essay_3'].apply(lambda x: 0 if x == 'unk' else 1)
 test['has_gt2_essays'] = test['project_essay_3'].apply(lambda x: 0 if x == 'unk' else 1 )<merge>
dis_text = train_data['text'][train_data['target']==1]
dis_text
resources['resources_total'] = resources['quantity'] * resources['price']
dfr = resources.groupby(['id'], as_index=False)[['resources_total']].sum()
train = pd.merge(train, dfr, how='left', on='id' ).fillna(-1)
 test = pd.merge(test, dfr, how='left', on='id' ).fillna(-1)
 dfr = resources.groupby(['id'], as_index=False)[['resources_total']].mean()
dfr = dfr.rename(columns={'resources_total':'resources_total_mean'})
 train = pd.merge(train, dfr, how='left', on='id' ).fillna(-1)
 test = pd.merge(test, dfr, how='left', on='id' ).fillna(-1)
 dfr = resources.groupby(['id'], as_index=False)[['quantity']].count()
dfr = dfr.rename(columns={'quantity':'resources_quantity_count'})
 train = pd.merge(train, dfr, how='left', on='id' ).fillna(-1)
 test = pd.merge(test, dfr, how='left', on='id' ).fillna(-1)
 dfr = resources.groupby(['id'], as_index=False)[['quantity']].sum()
dfr = dfr.rename(columns={'quantity':'resources_quantity_sum'})
 train = pd.merge(train, dfr, how='left', on='id' ).fillna(-1)
 test = pd.merge(test, dfr, how='left', on='id' ).fillna(-1)
 train = train.drop('id', axis=1)
 test = test.drop('id', axis=1 )<feature_engineering>
ndis_text = train_data['text'][train_data['target']==0]
ndis_text
train['project_essay'] = train.apply(lambda row: ' '.join([
str(row['project_title']),
str(row['project_essay_1']),
str(row['project_essay_2']),
str(row['project_essay_3']),
str(row['project_essay_4']),
str(row['project_resource_summary'])]), axis=1)
 test['project_essay'] = test.apply(lambda row: ' '.join([
str(row['project_title']),
str(row['project_essay_1']),
str(row['project_essay_2']),
str(row['project_essay_3']),
str(row['project_essay_4']),
str(row['project_resource_summary'])]), axis=1)
 train = train.drop([
'project_title',
'project_essay_1',
'project_essay_2',
'project_essay_3',
'project_essay_4',
'project_resource_summary'], axis=1)
 test = test.drop([
'project_title',
'project_essay_1',
'project_essay_2',
'project_essay_3',
'project_essay_4',
'project_resource_summary'], axis=1 )<string_transform>
dis_count = dis_text.map(lambda x: nltk.word_tokenize(x))
ndis_count = ndis_text.map(lambda x : nltk.word_tokenize(x))
dataset['text'] = dataset['text'].map(lambda x : nltk.word_tokenize(x))
ndis_count
w_tokenizer = nltk.tokenize.WhitespaceTokenizer()
lemmatizer = nltk.stem.WordNetLemmatizer()
def prep_text(text):
q = "['\’\´\ʻ]"
text = text.strip().lower()
text = re.sub('\W+',' ', text)
 text = re.sub(r'(")', ' ', text)
 text = re.sub(r"\\r|\
", " ", text)
 text = re.sub(re.compile("won%st" % q), "will not", text)
 text = re.sub(re.compile("can%st" % q), "can not", text)
 text = re.sub(re.compile("n%st" % q), " not", text)
 text = re.sub(re.compile("%sre" % q), " are", text)
 text = re.sub(re.compile("%ss" % q), " is", text)
 text = re.sub(re.compile("%sd" % q), " would", text)
 text = re.sub(re.compile("%sll" % q), " will", text)
 text = re.sub(re.compile("%st" % q), " not", text)
 text = re.sub(re.compile("%sve" % q), " have", text)
 text = re.sub(re.compile("%sm" % q), " am", text)
 text = [lemmatizer.lemmatize(w)for w in w_tokenizer.tokenize(text)]
return text
train['project_essay'] = train['project_essay'].apply(lambda x: prep_text(x))
test['project_essay'] = test['project_essay'].apply(lambda x: prep_text(x))<choose_model_class>
collection_words_dist = []
for i in list(dis_count):
collection_words_dist.extend(i)
 map_1 = FreqDist(collection_words_dist)
 map_1
tfv = TfidfVectorizer(norm='l2', min_df=0, max_features=8000,
strip_accents='unicode', analyzer='word', token_pattern=r'\w{1,}',
ngram_range=(1,2), use_idf=True, smooth_idf=False, sublinear_tf=True,
stop_words = 'english')
<categorify>
lemmatizer = WordNetLemmatizer()
def nltk_tags_2_word_tags(nltk_tag):
if nltk_tag.startswith('J'):
return wordnet.ADJ
elif nltk_tag.startswith('V'):
return wordnet.VERB
elif nltk_tag.startswith('N'):
return wordnet.NOUN
elif nltk_tag.startswith('R'):
return wordnet.ADV
else:
return None
def lemmatized_sentences(tokenized_sentence):
pos = nltk.pos_tag(tokenized_sentence)
 wordnet_tagged = map(lambda x:(x[0], nltk_tags_2_word_tags(x[1])) , pos)
 lemmatized_sentence = []
for word, tag in wordnet_tagged:
if tag is None:
lemmatized_sentence.append(word)
 else:
lemmatized_sentence.append(lemmatizer.lemmatize(word, tag))
return " ".join(lemmatized_sentence)
train_text = train['project_essay'].apply(lambda x: ' '.join(x))
test_text = test['project_essay'].apply(lambda x: ' '.join(x))
tfv.fit(list(train_text.values)+ list(test_text.values))
train_tfv = tfv.transform(train_text)
 test_tfv = tfv.transform(test_text)
 del train_text, test_text; gc.collect()<concatenate>
dis_lem = dis_count.apply(lemmatized_sentences)
 ndis_lem = ndis_count.apply(lemmatized_sentences)
 dataset['text'] = dataset['text'].apply(lemmatized_sentences)
 dataset
feat_train = train.drop('project_essay', axis=1)
 feat_test = test.drop('project_essay', axis=1)
 feat_train = csr_matrix(feat_train.values)
 feat_test = csr_matrix(feat_test.values)
 X_train_stack = hstack([feat_train, train_tfv[0:feat_train.shape[0]]])
 X_test_stack = hstack([feat_test, test_tfv[0:feat_test.shape[0]]])
 print('Train shape: ', X_train_stack.shape, '
Test Shape: ', X_test_stack.shape)
 del train, test, train_tfv, test_tfv; gc.collect()<define_variables>
def remove_pattern(input_txt, pattern):
reg_obj = re.compile(pattern)
 input_txt = reg_obj.sub(r'', input_txt)
 return input_txt
seed = 28<choose_model_class>
dataset['text'] = dataset['text'].apply(lambda x: remove_pattern(x,"@[\w]*"))
K = 5
kf = KFold(n_splits = K, random_state = seed, shuffle = True )<split>
dataset['text'] = dataset['text'].apply(lambda x: remove_pattern(x,'https?://\S+|www\.\S+'))
dataset['text'] = dataset['text'].apply(lambda x: remove_pattern(x,'<.*?>'))
cv_scores = []
xgb_preds = []
for train_index, test_index in kf.split(X_train_stack):
X_train, X_valid, y_train, y_valid = train_test_split(X_train_stack, target, test_size=0.20, random_state=random.seed(seed))
xgb_params = {'eta': 0.15,
'max_depth': 7,
'subsample': 0.80,
'colsample_bytree': 0.80,
'objective': 'binary:logistic',
'eval_metric': 'auc',
'seed': seed
}
d_train = xgb.DMatrix(X_train, y_train)
 d_valid = xgb.DMatrix(X_valid, y_valid)
 d_test = xgb.DMatrix(X_test_stack)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
model = xgb.train(xgb_params, d_train, 2000, watchlist, verbose_eval=50, early_stopping_rounds=30)
 cv_scores.append(float(model.attributes() ['best_score']))
xgb_pred = model.predict(d_test)
 xgb_preds.append(list(xgb_pred))
del X_train, X_valid, y_train, y_valid, d_train, d_valid, d_test; gc.collect()<define_variables>
dataset['text'] = dataset['text'].apply(lambda x: remove_pattern(x,"[^a-zA-Z
x_preds=[]
for i in range(len(xgb_preds[0])) :
sum=0
for j in range(K):
sum+=xgb_preds[j][i]
x_preds.append(sum / K )<save_to_csv>
tfidf = TfidfVectorizer(ngram_range =(1,1),max_df=0.90, min_df=2,stop_words = 'english')
 text_set = tfidf.fit_transform(dataset['text'])
 text_set
x_preds = pd.DataFrame(x_preds)
 x_preds.columns = ['project_is_approved']
submid = sub['id']
xsub = pd.concat([submid, x_preds], axis=1)
 xsub.to_csv('xgb_submission.csv', index=False )<split>
dataset_dtm = hstack(( text_set,np.array(dataset['location'])[:,None]))
cnt = 0
p_buf = []
n_splits = 5
n_repeats = 1
kf = RepeatedKFold(
 n_splits=n_splits,
n_repeats=n_repeats,
random_state=seed)
 auc_buf = []
for train_index, valid_index in kf.split(X_train_stack):
X_train, X_valid, y_train, y_valid = train_test_split(X_train_stack, target, test_size=0.20, random_state=random.seed(seed))
print('Fold {}/{}'.format(cnt + 1, n_splits))
params = {
'boosting_type': 'gbdt',
'objective': 'binary',
'metric': 'auc',
'max_depth': 7,
'num_leaves': 32,
'learning_rate': 0.02,
'feature_fraction': 0.80,
'bagging_fraction': 0.80,
'bagging_freq': 5,
'verbose': 0,
'lambda_l2': 1,
}
model = lgb.train(
 params,
lgb.Dataset(X_train, y_train),
num_boost_round=10000,
valid_sets=[lgb.Dataset(X_valid, y_valid)],
early_stopping_rounds=50,
verbose_eval=100 
)
 p = model.predict(X_valid, num_iteration=model.best_iteration)
 auc = roc_auc_score(y_valid, p)
 print('{} AUC: {}'.format(cnt, auc))
p = model.predict(X_test_stack, num_iteration=model.best_iteration)
 if len(p_buf)== 0:
p_buf = np.array(p)
 else:
p_buf += np.array(p)
 auc_buf.append(auc)
 cnt += 1
del model
gc.collect
auc_mean = np.mean(auc_buf)
 auc_std = np.std(auc_buf)
 print('AUC = {:.6f} +/- {:.6f}'.format(auc_mean, auc_std))
lgb_preds = p_buf/cnt
del X_valid, y_valid, X_train, y_train; gc.collect()<save_to_csv>
dataset_dtm=text_set
l_preds = pd.DataFrame(lgb_preds)
 l_preds.columns = ['project_is_approved']
l_preds.head()
submid = sub['id']
lsub = pd.concat([submid, l_preds], axis=1)
 lsub.to_csv('lgbm_submission.csv', index=False )<drop_column>
dataset_dtm = dataset_dtm.tocsr()
x_train = dataset_dtm[0:len(train_data)]
x_test = dataset_dtm[len(train_data):]
x_train.shape
del X_train_stack, X_test_stack; gc.collect()<load_from_csv>
y_train = train_data['target']
len(y_train )
train = pd.read_csv(data_path + 'donorschoose-application-screening/train.csv', dtype={"project_essay_3": object, "project_essay_4": object})
 labels = pd.DataFrame(train['project_is_approved'])
 labels.columns = ['project_is_approved']
train = train.drop('project_is_approved', axis=1)
 test = pd.read_csv(data_path + 'donorschoose-application-screening/test.csv', dtype={"project_essay_3": object, "project_essay_4": object})
 resources = pd.read_csv(data_path + 'donorschoose-application-screening/resources.csv')
 sub = pd.read_csv(data_path + 'donorschoose-application-screening/sample_submission.csv')
 train.fillna(( 'unk'), inplace=True)
 test.fillna(( 'unk'), inplace=True )<feature_engineering>
kfold = StratifiedKFold(n_splits = 5)
train['project_essay'] = train.apply(lambda row: ' '.join([
str(row['project_title']),
str(row['project_essay_1']),
str(row['project_essay_2']),
str(row['project_essay_3']),
str(row['project_essay_4']),
str(row['project_resource_summary'])]), axis=1)
 test['project_essay'] = test.apply(lambda row: ' '.join([
str(row['project_title']),
str(row['project_essay_1']),
str(row['project_essay_2']),
str(row['project_essay_3']),
str(row['project_essay_4']),
str(row['project_resource_summary'])]), axis=1)
 train = train.drop([
'project_title',
'project_essay_1',
'project_essay_2',
'project_essay_3',
'project_essay_4',
'project_resource_summary'], axis=1)
 test = test.drop([
'project_title',
'project_essay_1',
'project_essay_2',
'project_essay_3',
'project_essay_4',
'project_resource_summary'], axis=1)
 gc.collect()<feature_engineering>
random_state = 2
classifiers = []
classifiers.append(SVC(kernel = 'rbf',probability = True))
classifiers.append(DecisionTreeClassifier(random_state=random_state))
classifiers.append(RandomForestClassifier(random_state=random_state))
classifiers.append(ExtraTreesClassifier(random_state=random_state))
classifiers.append(GradientBoostingClassifier(random_state=random_state))
classifiers.append(MLPClassifier(random_state=random_state))
classifiers.append(KNeighborsClassifier())
 classifiers.append(LogisticRegression())
 classifiers.append(XGBClassifier(random_state=random_state))
COMMENT = 'project_essay'
re_tok = re.compile(f'([{string.punctuation}“”¨«»®´·º½¾¿¡§£₤‘’])')
 def tokenize(s): return re_tok.sub(r' \1 ', s ).split()
n = train.shape[0]
vec = TfidfVectorizer(ngram_range=(1,3), tokenizer=tokenize,
min_df=3, max_df=0.95, strip_accents='unicode', use_idf=1,
smooth_idf=1, sublinear_tf=1)
 train_vec = vec.fit_transform(train[COMMENT])
 test_vec = vec.transform(test[COMMENT])
 train = train.drop('project_essay', axis=1)
 test = test.drop('project_essay', axis=1 )<train_model>
parameter = {'solver':['liblinear','lbfgs'],
'max_iter':[200,400]}
Logis_clf = LogisticRegression()
lreg = GridSearchCV(Logis_clf, param_grid = parameter, cv = 3, verbose=True, n_jobs=-1)
 lreg.fit(x_train, y_train)
 lreg_best = lreg.best_estimator_
print(lreg.best_score_)
 print(lreg.best_params_ )
label_cols = ['project_is_approved']
x = train_vec
test_x = test_vec
def pr(y_i, y):
p = x[y==y_i].sum(0)
 return(p+1)/(( y==y_i ).sum() +1)
 def get_mdl(y):
y = y.values
r = np.log(pr(1,y)/ pr(0,y))
m = LogisticRegression(C=4, dual=True)
 x_nb = x.multiply(r)
 return m.fit(x_nb, y), r
nb_preds = np.zeros(( test.shape[0], len(label_cols)))
 for i, j in enumerate(label_cols):
print('fitting: ', j)
 m,r = get_mdl(labels[j])
 nb_preds[:,i] = m.predict_proba(test_x.multiply(r)) [:,1]<save_to_csv>
S_train, S_test = stacking(classifiers,
x_train, y_train, x_test,
regression= False,
mode='oof_pred_bag',
needs_proba=True,
save_dir=None,
n_folds=5,
stratified=True,
shuffle=True,
random_state=0,
verbose=2 )
nb_preds = pd.DataFrame(nb_preds)
 nb_preds.columns = ['project_is_approved']
submid = sub['id']
nbsub = pd.concat([submid, nb_preds], axis=1)
 nbsub.to_csv('nb_submission.csv', index=False )<rename_columns>
argmax_train = []
argmax_test = []
for i in range(0,S_train.shape[1],2):
argmax_train.append(np.argmax(S_train[:,i:i+2],axis=1))
argmax_test.append(np.argmax(S_test[:,i:i+2],axis=1))
almost_zero = 1e-10
almost_one = 1-almost_zero
nbsub.columns = ['id', 'project_is_approved1']
df1 = nbsub
lsub.columns = ['id', 'project_is_approved2']
df2 = lsub
xsub.columns = ['id', 'project_is_approved3']
df3 = xsub
df = pd.merge(df1, df2, on='id')
 df = pd.merge(df, df3, on='id')
 print(df.head())
 scores = [0, 0.72237, 0.77583, 0.77212]
wts = [0, 0, 0, 0]
power = 68
wts[1] = scores[1] ** power
wts[2] = scores[2] ** power
wts[3] = scores[3] ** power
print(wts[:])
 number1 = df['project_is_approved1'].clip(almost_zero,almost_one ).apply(logit)* wts[1]
number2 = df['project_is_approved2'].clip(almost_zero,almost_one ).apply(logit)* wts[2]
number3 = df['project_is_approved3'].clip(almost_zero,almost_one ).apply(logit)* wts[3]
totweight = wts[0] + wts[1] + wts[2] + wts[3]
df['project_is_approved'] =(number1 + number2 + number3)/(totweight)
 df['project_is_approved'] = df['project_is_approved'].apply(expit)
 df[['id', 'project_is_approved']].to_csv("ensembling-logit_submission.csv", index=False )<set_options>
argmax_train = np.array(argmax_train,dtype= np.int64 ).T
argmax_test = np.array(argmax_test,dtype= np.int64 ).T
warnings.filterwarnings('ignore' )<load_from_csv>
modelc = LogisticRegression()
model1c = modelc.fit(S_train, y_train)
 y_pred1c = model1c.predict_proba(S_train)
 y_predc = model1c.predict_proba(S_test)
 print('Final test prediction score: [%.8f]' % accuracy_score(y_train, np.argmax(y_pred1,axis=1)))
 print('Final f1-score test prediction: [%.8f]' % f1_score(y_train, np.argmax(y_pred1,axis=1)))
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )<define_variables>
model = XGBClassifier(random_state=2, objective = 'reg:linear', n_jobs=-1, learning_rate= 0.5,
n_estimators=30, max_depth=20)
 model1 = model.fit(argmax_train, y_train)
 y_pred1 = model1.predict_proba(argmax_train)
 y_pred = model1.predict_proba(argmax_test)
 print('Final test prediction score: [%.8f]' % accuracy_score(y_train, np.argmax(y_pred1,axis=1)))
 print('Final f1-score test prediction: [%.8f]' % f1_score(y_train, np.argmax(y_pred1,axis=1)))
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]<set_options>
sample_submission['target'] = np.argmax(y_pred+y_predc,axis=1)
 sample_submission.to_csv('submission_with_stacking.csv', index = False )