Kaggle competition: Recursion Cellular Image Classification
<SOS> metric: CategorizationAccuracy Kaggle data source: recursion-cellular-image-classification<compute_train_metric>
model_id = 'ICN2247013813647982800'
score_threshold = 0.000001
gcp_service_account_json = '/kaggle/input/gcloudserviceaccountkey/kaggle-playground-170215-4ece6a076f22.json'
gcp_project_id = 'kaggle-playground-170215'
seed=1029
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in tqdm([i * 0.01 for i in range(100)], disable=True):
score = f1_score(y_true=y_true, y_pred=y_proba > threshold)
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'f1': best_score}
return search_result
def seed_everything(seed=1234):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()<prepare_x_and_y>
!pip install google-cloud-automl
train_preds = np.zeros(len(train))
test_preds = np.zeros(( len(test), len(splits)))
 n_epochs = 5
for i,(train_idx, valid_idx)in enumerate(splits):
x_train_fold = torch.tensor(X_train[train_idx], dtype=torch.long ).cuda()
y_train_fold = torch.tensor(y_train[train_idx, np.newaxis], dtype=torch.float32 ).cuda()
x_val_fold = torch.tensor(X_train[valid_idx], dtype=torch.long ).cuda()
y_val_fold = torch.tensor(y_train[valid_idx, np.newaxis], dtype=torch.float32 ).cuda()
train = torch.utils.data.TensorDataset(x_train_fold, y_train_fold)
 valid = torch.utils.data.TensorDataset(x_val_fold, y_val_fold)
 train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)
 print(f'Fold {i + 1}')
 seed_everything(seed + i)
 model = NeuralNet()
model.cuda()
valid_preds_fold, test_preds_fold = train_model(model,
x_train_fold,
y_train_fold,
x_val_fold,
y_val_fold, validate=False)
 train_preds[valid_idx] = valid_preds_fold
test_preds[:, i] = test_preds_fold<save_to_csv>
from concurrent.futures import ProcessPoolExecutor as PoolExecutor, as_completed
from google.cloud import automl_v1beta1
from tqdm import tqdm
import operator
import os
import pandas as pd
import sys
import time
search_result = threshold_search(y_train, train_preds)
 sub['prediction'] = test_preds.mean(1)> search_result['threshold']
sub.to_csv("submission.csv", index=False )<import_modules>
def get_prediction(file_path, project_id, model_id):
name = 'projects/{}/locations/us-central1/models/{}'.format(project_id, model_id)
 with open(file_path, 'rb')as ff:
content = ff.read()
payload = {'image': {'image_bytes': content }}
params = {'score_threshold':str(score_threshold)}
request = prediction_client.predict(name, payload, params)
 return request
import numpy as np
import pandas as pd
import os
from multiprocessing import Pool<load_from_csv>
def make_int(s):
try:
int(s)
 return int(s)
 except ValueError:
return 1109
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape )<concatenate>
def process(i,df_sample_submission,project_id):
id_code = df_sample_submission.index[i]
if id_code in df_solution.index:
return None
exp_len = id_code.find('_')
 experiment = id_code[0:exp_len]
plate = id_code[(exp_len+1):(exp_len+2)]
well = id_code[exp_len+3:]
pred_dict = {}
res = []
for site in range(1,3):
file_path = '.. /input/recursion_rgb_512/testrgb512/testRGB512/{}_{}_{}_s{}.png'.format(experiment,plate,well,site)
 try:
prediction_request = get_prediction(file_path, project_id, model_id)
 except Exception as e:
print('Something got wrong: ', e)
 return None
for prediction in prediction_request.payload:
label = make_int(prediction.display_name)
 if label <= 1108:
pred_dict[label] = float(prediction.classification.score)
 sirna_prediction = max(pred_dict.items() , key=operator.itemgetter(1)) [0]
confidence = pred_dict[sirna_prediction]
res.append({
'id_code': id_code,
'site': site,
'sirna_prediction': sirna_prediction,
'confidence': confidence
})
 return res
def parallelize_apply(df,func,colname,num_process,newcolnames):
pool =Pool(processes=num_process)
 arraydata = pool.map(func,tqdm(df[colname].values))
pool.close()
newdf = pd.DataFrame(arraydata,columns = newcolnames)
 df = pd.concat([df,newdf],axis=1)
 return df
def parallelize_dataframe(df, func):
df_split = np.array_split(df, 4)
 pool = Pool(4)
 df = pd.concat(pool.map(func, df_split))
pool.close()
pool.join()
return df
def add_features(df):
df['question_text'] = df['question_text'].apply(lambda x:str(x))
df['total_length'] = df['question_text'].apply(len)
 df['capitals'] = df['question_text'].apply(lambda comment: sum(1 for c in comment if c.isupper()))
df['caps_vs_length'] = df.apply(lambda row: float(row['capitals'])/float(row['total_length']),
axis=1)
 df['num_words'] = df.question_text.str.count('\S+')
 df['num_unique_words'] = df['question_text'].apply(lambda comment: len(set(w for w in comment.split())))
 df['words_vs_unique'] = df['num_unique_words'] / df['num_words']
return df 
<create_dataframe>
def generated_predictions_with_pool_executor(max_workers,gcp_project_id):
results = []
df_sample_submission = pd.read_csv('.. /input/recursion-cellular-image-classification/sample_submission.csv',index_col=[0])
 with PoolExecutor(max_workers=max_workers)as executor:
futures_list = [executor.submit(process, i,df_sample_submission,gcp_project_id)for i in range(len(df_sample_submission)) ]
for f in tqdm(as_completed(futures_list), total=len(futures_list)) :
results.append(f.result())
 nb_escaped = 0
for r in results:
if r is None:
nb_escaped += 1
continue
for site in r:
df_solution.loc[site['id_code'], ['site{}_sirna'.format(site['site']),'site{}_confidence'.format(site['site'])]] = [site['sirna_prediction'], site['confidence']]
train_df = parallelize_dataframe(train_df, add_features)
 test_df = parallelize_dataframe(test_df, add_features )<string_transform>
solution_file_path = ',/submissions/submission_{}.csv'.format(model_id)
 if os.path.exists(solution_file_path):
df_solution = pd.read_csv(solution_file_path,index_col=[0])
 else:
df_solution = pd.DataFrame(columns=['site1_sirna','site1_confidence','site2_sirna','site2_confidence'])
 df_solution.index.name = 'id_code'
prediction_client = automl_v1beta1.PredictionServiceClient.from_service_account_json(gcp_service_account_json)
 generated_predictions_with_pool_executor(20,gcp_project_id)
 generated_predictions_with_pool_executor(5,gcp_project_id)
 generated_predictions_with_pool_executor(5,gcp_project_id)
max_features = 100000
tokenizer = Tokenizer(num_words=max_features, lower=True)
 full_text = list(train_df['question_text'].values)
 tokenizer.fit_on_texts(full_text)
 train_tokenized = tokenizer.texts_to_sequences(train_df['question_text'].fillna('missing'))
test_tokenized = tokenizer.texts_to_sequences(test_df['question_text'].fillna('missing'))<prepare_x_and_y>
site1_wins = df_solution['site1_confidence'] >= df_solution['site2_confidence']
site2_wins = df_solution['site2_confidence'] > df_solution['site1_confidence']
df_solution.loc[site1_wins,'sirna'] =(df_solution[site1_wins])['site1_sirna']
df_solution.loc[site2_wins,'sirna'] =(df_solution[site2_wins])['site2_sirna']
df_solution['sirna'] = df_solution['sirna'].astype(int)
 df_solution['sirna'].sort_index().to_csv('submit_{}.csv'.format(model_id),header=True)