Kaggle competition: TReNDS Neuroimaging
%matplotlib inline
rcParams['figure.figsize'] = [10,5]
plt.style.use('fivethirtyeight')
 sns.set_style('whitegrid')
 warnings.filterwarnings('ignore')
 pd.set_option('display.max_rows', 50)
 pd.set_option('display.max_columns', 150)
 pd.options.display.float_format = '{:.4f}'.format<install_modules>
from fastai.vision import *
!pip install googletrans
!pip install p_tqdm
!pip install transformers==3.0.2
clear_output()<load_from_csv>
pip install chainer
df_train = pd.read_csv('.. /input/contradictory-my-dear-watson/train.csv')
 df_test = pd.read_csv('.. /input/contradictory-my-dear-watson/test.csv' )<categorify>
pip install cupy --no-cache-dir -vvvv
translator = Translator()
def translate_to_en(text):
global translator
translated_text = translator.translate(text, dest = 'en' ).text
return translated_text
def pool_translated_en(list_text):
threads = 8*multiprocessing.cpu_count()
with Pool(threads)as executor:
result = list(tqdm(executor.imap(translate_to_en, list_text), total=len(list_text)))
 return result
def dataset_translated_en(df):
df_trans = df.copy()
list_idx = list(df_trans[df_trans['lang_abv']!='en'].index)
 list_prem = list(df_trans.loc[list_idx]['premise'])
 list_hyp = list(df_trans.loc[list_idx]['hypothesis'])
 print('
Premise Translation - En')
 df_trans.loc[list_idx,'premise'] = pool_translated_en(list_prem)
 print('
Premise Hypotheses - En')
 df_trans.loc[list_idx,'hypothesis'] = pool_translated_en(list_hyp)
 df_trans['lang_abv'] = 'en'
df_trans['language'] = 'English'
return df_trans
df_train_en = dataset_translated_en(df_train)
 df_test_en = dataset_translated_en(df_test )<filter>
def metric(y_true, y_pred):
return np.mean(np.sum(np.abs(y_true - y_pred), axis=0)/np.sum(y_true, axis=0))
list_non_en_idx = list(df_train[df_train['lang_abv']!='en'].index)
 for i in range(5):
idx = random.randint(0,len(list_non_en_idx))
print('
ORIGINAL TEXT :',df_train['premise'][list_non_en_idx[idx]])
 print('TRANSLATED TEXT :',df_train_en['premise'][list_non_en_idx[idx]] )<count_values>
fnc_df = cudf.read_csv(".. /input/trends-assessment-prediction/fnc.csv")
 loading_df = cudf.read_csv(".. /input/trends-assessment-prediction/loading.csv")
 fnc_features, loading_features = list(fnc_df.columns[1:]), list(loading_df.columns[1:])
 df = fnc_df.merge(loading_df, on="Id")
 labels_df = cudf.read_csv(".. /input/trends-assessment-prediction/train_scores.csv")
 labels_df["is_train"] = True
df = df.merge(labels_df, on="Id", how="left")
 test_df = df[df["is_train"] != True].copy()
df = df[df["is_train"] == True].copy()
df.shape, test_df.shape
print('Proportion for each languange in train dataset')
 df_train['language'].value_counts() / len(df_train)* 100<feature_engineering>
FNC_SCALE = 1/500
df[fnc_features] *= FNC_SCALE
test_df[fnc_features] *= FNC_SCALE
def remove_punctuation(text):
no_punct = ''.join([c for c in text if c not in string.punctuation])
 return no_punct
def quick_clean_data(dataset, var):
df = dataset.copy()
df[var] = df[var].str.lower()
df[var] = df[var].str.strip()
df[var] = df.apply(lambda x : remove_punctuation(x[var]), axis=1)
 df[var] = df.apply(lambda x : " ".join(x[var].split()), axis=1)
 return df
list_var = ['premise','hypothesis']
for var in list_var :
df_train = quick_clean_data(df_train, var)
 df_test = quick_clean_data(df_test, var)
 df_train_en = quick_clean_data(df_train_en, var)
 df_test_en = quick_clean_data(df_test_en, var )<string_transform>
NUM_FOLDS = 7
kf = KFold(n_splits=NUM_FOLDS, shuffle=True, random_state=0)
 features = loading_features + fnc_features
overal_score = 0
for target, c, w, ff in [("age", 60, 0.3, 0.55),("domain1_var1", 12, 0.175, 0.2),("domain1_var2", 8, 0.175, 0.2),("domain2_var1", 9, 0.175, 0.22),("domain2_var2", 12, 0.175, 0.22)]:
y_oof = np.zeros(df.shape[0])
 y_test = np.zeros(( test_df.shape[0], NUM_FOLDS))
for f,(train_ind, val_ind)in enumerate(kf.split(df, df)) :
train_df, val_df = df.iloc[train_ind], df.iloc[val_ind]
train_df = train_df[train_df[target].notnull() ]
model_1 = SVR(C=c, cache_size=3000.0)
 model_1.fit(train_df[features].values, train_df[target].values)
 model_2 = Ridge(alpha = 0.0001)
 model_2.fit(train_df[features].values, train_df[target].values)
 val_pred_1 = model_1.predict(val_df[features])
 val_pred_2 = model_2.predict(val_df[features])
 test_pred_1 = model_1.predict(test_df[features])
 test_pred_2 = model_2.predict(test_df[features])
 val_pred =(1-ff)*val_pred_1+ff*val_pred_2
val_pred = cp.asnumpy(val_pred.values.flatten())
 test_pred =(1-ff)*test_pred_1+ff*test_pred_2
test_pred = cp.asnumpy(test_pred.values.flatten())
 y_oof[val_ind] = val_pred
y_test[:, f] = test_pred
df["pred_{}".format(target)] = y_oof
test_df[target] = y_test.mean(axis=1)
 score = metric(df[df[target].notnull() ][target].values, df[df[target].notnull() ]["pred_{}".format(target)].values)
 mae = mean_absolute_error(df[df[target].notnull() ][target].values, df[df[target].notnull() ]["pred_{}".format(target)].values)
 rmse = np.sqrt(mean_squared_error(df[df[target].notnull() ][target].values, df[df[target].notnull() ]["pred_{}".format(target)].values))
overal_score += w*score
print(target, np.round(score, 8))
print(target, np.round(score, 4))
print(target, 'mean absolute error:', np.round(mae, 8))
print(target, ' root mean square error:', np.round(rmse, 8))
print()
print("Overal score:", np.round(overal_score, 8))
print("Overal score:", np.round(overal_score, 4))
tqdm_notebook.pandas()
def remove_stop_words(text):
en_stop_words = stopwords.words('english')
 text = ' '.join([c for c in text.split() if c not in en_stop_words])
 return text
df_train_no_stop = df_train_en.copy()
df_test_no_stop = df_test_en.copy()
list_var = ['premise','hypothesis']
for var in list_var :
df_train_no_stop[var] = df_train_no_stop.progress_apply(lambda x : remove_stop_words(x[var]), axis=1)
 df_test_no_stop[var] = df_test_no_stop.progress_apply(lambda x : remove_stop_words(x[var]), axis=1 )<define_variables>
sub_df = cudf.melt(test_df[["Id", "age", "domain1_var1", "domain1_var2", "domain2_var1", "domain2_var2"]], id_vars=["Id"], value_name="Predicted")
 sub_df["Id"] = sub_df["Id"].astype("str")+ "_" + sub_df["variable"].astype("str")
 sub_df = sub_df.drop("variable", axis=1 ).sort_values("Id")
 assert sub_df.shape[0] == test_df.shape[0]*5
sub_df.head(10 )
<define_variables><EOS> 
sub_df.to_csv("submission_rapids_ensemble.csv", index=False )