Kaggle competition: Costa Rican Household Poverty Level Prediction
<SOS> metric: MacroFScore Kaggle data source: costa-rican-household-poverty-level-prediction<choose_model_class>
%matplotlib inline
model_checkpoint = tf.keras.callbacks.ModelCheckpoint('model.h5', monitor='val_accuracy', verbose=2, save_best_only=True)
 histories = Tensordash(
 email = secret_value_0,
password = secret_value_1,
ModelName = "Plant Disease Model" )<train_model>
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv')
 print('train data shape :',train_df.shape)
 print('test data shape :',test_df.shape )
model_history = model.fit_generator(train_generator, epochs=EPOCHS, validation_data=valid_generator, callbacks=[model_checkpoint,lr_callback, histories] )<save_to_csv>
y=train_df['Target']
y.value_counts()
pd.DataFrame(model_history.history ).to_csv('ModelHistory.csv' )<load_from_csv>
train_null = train_df.isnull().sum()
train_null[train_null > 0]
test_dataset = pd.read_csv('/kaggle/input/plant-pathology-2020-fgvc7/test.csv')
 submission = pd.read_csv('/kaggle/input/plant-pathology-2020-fgvc7/sample_submission.csv')
 test_dataset<feature_engineering>
test_null = test_df.isnull().sum()
test_null[test_null > 0]
test_dataset['image_id'] = test_dataset['image_id']+'.jpg'<create_dataframe>
train_grphh = train_df.groupby('idhogar')['Target'].apply(lambda x: x.nunique() == 1)
 err_train = train_grphh[train_grphh !=True]
print('Number of households with incorrect poverty level :',len(err_train))
test_gen = datagen.flow_from_dataframe(test_dataset,
directory='/kaggle/input/plant-pathology-2020-fgvc7/images/',
x_col='image_id',
target_size=(512, 512),
class_mode=None,
shuffle=False,
batch_size=8 )<predict_on_test>
for household in err_train.index:
target = int(train_df[(train_df['idhogar']==household)&(train_df['parentesco1']==1.0)]['Target'])
 train_df.loc[train_df['idhogar']==household,'Target'] = target
predictions = model.predict_generator(test_gen )<define_variables>
train_df,test_df = train_df.align(test_df,join='inner',axis=1)
 print('Training Features shape: ', train_df.shape)
 print('Testing Features shape: ', test_df.shape )
submission['healthy'] = predictions[:, 0]
submission['multiple_diseases'] = predictions[:, 1]
submission['rust'] = predictions[:, 2]
submission['scab'] = predictions[:, 3]<save_to_csv>
data = pd.concat([train_df,test_df],axis=0)
 data.head()
submission.to_csv('submission.csv', index=False )<save_to_csv>
submission.to_csv('submission.csv', index=False )<set_options>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<categorify>
def regular_encode(texts, tokenizer, maxlen=512):
enc_di = tokenizer.batch_encode_plus(
 texts,
return_attention_masks=False,
return_token_type_ids=False,
pad_to_max_length=True,
max_length=maxlen 
)
 return np.array(enc_di['input_ids'] )<choose_model_class>
cat_cols = data.nunique() ==2
cat_cols = list(cat_cols[cat_cols].keys() )
def build_model(transformer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_lang_tags = Input(shape=(4,), dtype=tf.float32, name="input_lang_tags")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
x = Concatenate()([cls_token, input_lang_tags])
 out = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=[input_word_ids, input_lang_tags], outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=[tf.keras.metrics.AUC() ])
 return model 
<define_variables>
cols = ['edjefe', 'edjefa','dependency']
data[cols] = data[cols].replace({'no': 0, 'yes':1} ).astype(float)
 data['hogar_mid'] = data['hogar_adul'] - data['hogar_mayor']
data['bedroom%'] = data['bedrooms']/data['rooms']
data['person/rooms'] = data['rooms']/data['hhsize']
data['male_ratio'] = data['r4h3']/data['r4t3']
data['female_ratio'] = data['r4m3']/data['r4t3']
data['female_per_room'] = data['r4m3']/data['rooms']
data['female_per_bedroom'] = data['r4m3']/data['bedrooms']
data['hogarmid_per_bedroom'] = data['hogar_mid']/data['bedrooms']
data['escolari_age'] = data['escolari']/data['age']
data['hogar_nin_per_room'] = data['rooms']/(data['hogar_nin']+1)
 data['male_per_bedroom'] = data['r4h3']/data['bedrooms']
data['male_per_room'] = data['r4h3']/data['rooms']
data['dependencyXmale']= data['dependency']*data['r4h3']
data['dependencyXhogar_mid'] = data['dependency']*data['hogar_mid']
data['dependencyXhogar_adul'] = data['dependency']*data['hogar_adul']
data['dependency_prod_sum']=data['dependencyXmale'] + data['dependencyXhogar_mid'] + data['dependencyXhogar_adul']
data['overcrowding_room_and_bedroom'] =(data['hacdor'] + data['hacapo'])/2
data['no_appliances'] = data['refrig'] + data['computer'] + data['television']
AUTO = tf.data.experimental.AUTOTUNE
EPOCHS = 2
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
MAX_LEN = 192<load_pretrained>
df_group = pd.DataFrame()
other_list = ['escolari', 'age', 'escolari_age']
for item in other_list:
for function in ['mean','std','min','max','sum']:
group_data = data[item].groupby(data['idhogar'] ).agg(function)
 new_col = item + '_' + function
df_group[new_col] = group_data
df_group = df_group.reset_index()
data = pd.merge(data, df_group, on='idhogar' )
MODEL = 'jplu/tf-xlm-roberta-large'
tokenizer = AutoTokenizer.from_pretrained(MODEL )<categorify>
data.fillna(-1,inplace=True)
def lang_embed(lang, tran):
lang_codes = {'en':'000', 'es':'100', 'fr':'010',
'it':'001', 'pt':'110', 'ru':'101',
'tr':'011'}
tran_codes = {'orig':'0', 'tran':'1'}
vec = lang_codes[lang]+tran_codes[tran]
vec = [int(v)for v in vec]
return vec<load_from_csv>
scorer = make_scorer(f1_score, greater_is_better=True, average = 'macro' )
def text_process(text):
ws = text.split(' ')
 if(len(ws)>160):
text = ' '.join(ws[:160])+ ' ' + ' '.join(ws[-32:])
 return text
valid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')
 valid['comment_text'] = valid['comment_text'].apply(lambda x: text_process(x))
x_valid = regular_encode(valid.comment_text.values, tokenizer, maxlen=MAX_LEN)
 lang_tag_valid = np.array([lang_embed(row['lang'], 'orig')for _, row in valid.iterrows() ])
 y_valid = valid.toxic.values
gap = 128
x_valid = np.concatenate(( x_valid, x_valid[-gap:]))
lang_tag_valid = np.concatenate(( lang_tag_valid, lang_tag_valid[-gap:]))
y_valid = np.concatenate(( y_valid, y_valid[-gap:]))
print(y_valid.shape)

sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv' )<load_from_csv>
def macro_f1_score(labels, predictions):
predictions = predictions.reshape(len(np.unique(labels)) , -1 ).argmax(axis = 0)
 metric_value = f1_score(labels, predictions, average = 'macro')
 return 'macro_f1', metric_value, True
test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 test['content'] = test['content'].apply(lambda x: text_process(x))
x_test = regular_encode(test.content.values, tokenizer, maxlen=MAX_LEN)
 lang_tag_test = np.array([lang_embed(row['lang'], 'orig')for _, row in test.iterrows() ])
<load_from_csv>
X = train_df
logR = LogisticRegression(class_weight='balanced',C=0.0005)
 cv_score = cross_val_score(logR, X, y, cv = 10, scoring = scorer)
 print(f'10 Fold Cross Validation F1 Score = {round(cv_score.mean() , 4)} with std = {round(cv_score.std() , 4)}')
mybest = pd.read_csv('/kaggle/input/mybest/sub9521.csv')
 mybest['orig'] = test['content']
mybest['lang'] = test['lang']
mybest['tran'] = ''
out = []
for _, row in mybest.iterrows() :
if(row['toxic']>=0.5):
out.append([row['orig'], row['tran'], row['lang'], 1])
 elif(row['toxic']<0.5):
out.append([row['orig'], row['tran'], row['lang'], 0])
 train = pd.DataFrame(out, columns=['orig', 'tran', 'lang', 'toxic'])
 train['orig'] = train['orig'].apply(lambda x: text_process(x))
print(train.shape )<prepare_x_and_y>
logR.fit(X,y)
 preds_log = logR.predict(test_df)
 sub_log = pd.DataFrame({'Id':test_df.index, 'Target':preds_log})
 sub_log.to_csv('sub_log1.csv', index=False)