Kaggle competition: Recursion Cellular Image Classification
sequences = create_sequences(data, stats)
 sequences = np.array(sequences)
 sequences_train = np.delete(sequences, [0], 2)
 sequences_train = np.array(sequences_train )<feature_engineering>
LABELS = ["sirna"]
all_files = glob.glob(".. /input/cellstack/*.csv")
 all_files
def dain(input):
n_features = 2
mean = Lambda(lambda x: K.mean(input, axis=1))(input)
 adaptive_avg = Dense(n_features,
kernel_initializer=Identity(gain=1.0),
bias=False )(mean)
 adaptive_avg = Reshape(( 1, n_features))(adaptive_avg)
 X = Lambda(lambda inputs: inputs[0] - inputs[1] )([input, adaptive_avg])
 std = Lambda(lambda x: K.mean(x**2, axis=1))(X)
 std = Lambda(lambda x: K.sqrt(x+1e-8))(std)
 adaptive_std = Dense(n_features,
bias=False )(std)
 adaptive_std = Reshape(( 1, n_features))(adaptive_std)
 X = Lambda(lambda inputs: inputs[0] / inputs[1] )([X, adaptive_std])
 avg = Lambda(lambda x: K.mean(x, axis=1))(X)
 gate = Dense(n_features,
activation="sigmoid",
kernel_initializer=Identity(gain=1.0),
bias=False )(avg)
 gate = Reshape(( 1, n_features))(gate)
 X = Lambda(lambda inputs: inputs[0] * inputs[1] )([X, gate])
 return X, adaptive_avg, adaptive_std<categorify>
outs = [pd.read_csv(f, index_col=0)for f in all_files]
concat_sub = pd.concat(outs, axis=1)
 cols = list(map(lambda x: "m" + str(x), range(len(concat_sub.columns))))
concat_sub.columns = cols
concat_sub.reset_index(inplace=True )
def build_generator(encoder_input_shape, missing_len, verbose=True):
learning_rate = 0.0002
optimizer = Adam(lr=learning_rate)
 generator_decoder_type ='seq2seq'
encoder_inputs = Input(shape=encoder_input_shape)
 hidden, avg, std = dain(encoder_inputs)
 decoder_outputs = []
encoder = LSTM(128, return_sequences=True, return_state=True)
 lstm_outputs, state_h, state_c = encoder(hidden)
 if generator_decoder_type == 'seq2seq':
states = [state_h, state_c]
decoder_lstm = LSTM(128, return_sequences=True, return_state=True)
 decoder_cases = Dense(1, activation='relu')
 decoder_deaths = Dense(1, activation='relu')
 all_outputs_c = []
all_outputs_d = []
inputs = lstm_outputs
for idx in range(missing_len):
outputs, state_h, state_c = decoder_lstm(inputs, initial_state=states)
 inputs = outputs
outputs = BatchNormalization()(outputs)
 outputs = Flatten()(outputs)
 outputs_cases = decoder_cases(outputs)
 outputs_deaths = decoder_deaths(outputs)
 states = [state_h, state_c]
std_c = Lambda(lambda inputs: inputs[:, 0, 0] )(std)
 avg_c = Lambda(lambda inputs: inputs[:, 0, 0] )(avg)
 outputs_cases = Multiply()([outputs_cases, std_c])
 outputs_cases = Add()([outputs_cases, avg_c])
 std_d = Lambda(lambda inputs: inputs[:, 0, 1] )(std)
 avg_d = Lambda(lambda inputs: inputs[:, 0, 1] )(avg)
 outputs_deaths = Multiply()([outputs_deaths, std_d])
 outputs_deaths = Add()([outputs_deaths, avg_d])
 all_outputs_c.append(outputs_cases)
 all_outputs_d.append(outputs_deaths)
 decoder_outputs_c = Lambda(lambda x: x )(outputs_cases)
 decoder_outputs_d = Lambda(lambda x: x )(outputs_deaths)
 model = Model(inputs=encoder_inputs,
outputs=[decoder_outputs_c, decoder_outputs_d])
 if verbose:
print('
Generator summary: ')
 print(model.summary())
 model.compile(loss='mean_squared_logarithmic_error', optimizer=optimizer)
 return model<predict_on_test>
rank = np.tril(concat_sub.iloc[:,1:].corr().values,-1)
 m =(rank>0 ).sum()
m_gmean, s = 0, 0
for n in range(min(rank.shape[0],m)) :
mx = np.unravel_index(rank.argmin() , rank.shape)
 w =(m-n)/(m+n/10)
 print(w)
 m_gmean += w*(np.log(concat_sub.iloc[:,mx[0]+1])+np.log(concat_sub.iloc[:,mx[1]+1])) /2
s += w
rank[mx] = 1
m_gmean = np.exp(m_gmean/s ).clip(0.0,1.0 )
def backtest2(sequences, model, given, missing):
sequences_test = sequences[:, -given:]
pred_d = []
pred_c = []
for i in range(0, missing):
predictions = model.predict(sequences_test[:, :, ])
 predictions[0][predictions[0] < 0] = 0
predictions[1][predictions[1] < 0] = 0
predictions[1] = np.around(predictions[1].astype(np.double))
predictions[0] = np.around(predictions[0].astype(np.double))
pred = np.concatenate(
 [np.expand_dims(predictions[0], axis=2),
np.expand_dims(predictions[1], axis=2)],
axis=2)
 pred_c.append(pred)
 pred_d.append(predictions[1])
 sequences_test = np.concatenate([sequences_test[:, 1:],
pred],
axis=1)
 predictions = np.array(pred_c[0])
 for i in range(1, len(pred_c)) :
predictions = np.concatenate([predictions,
pred_c[i]],
axis=1)
 seq_cases = sequences[:, :, 0]
seq_death = sequences[:, :, 1]
death = np.cumsum(seq_death, axis=1)
 cases = np.cumsum(seq_cases, axis=1)
 cases = np.around(cases.astype(np.double))
cases[cases < 0] = 0
cases_csv = np.expand_dims(cases[:, -1], axis=1)
 predictions[0] = np.around(predictions[0].astype(np.double))
cases_csv = np.concatenate(( cases_csv, predictions[:, :, 0]), axis=1)
 death = np.around(death.astype(np.double))
death[death < 0] = 0
death_csv = np.expand_dims(death[:, -1], axis=1)
 predictions[1] = np.around(predictions[1].astype(np.double))
death_csv = np.concatenate(( death_csv, predictions[:, :, 1]), axis=1)
 cases_csv = np.cumsum(cases_csv, axis=1)
 death_csv = np.cumsum(death_csv, axis=1)
 death_csv = death_csv[:, 1:]
cases_csv = cases_csv[:, 1:]
death_csv = np.concatenate(( death[:, -11:], death_csv), axis=1)
 cases_csv = np.concatenate(( cases[:, -11:], cases_csv), axis=1)
 csv = []
cases_csv = np.reshape(cases_csv[:, 1:],(-1, 1))
death_csv = np.reshape(death_csv[:, 1:],(-1, 1))
j = 1
for idx,(c, d)in enumerate(zip(cases_csv, death_csv)) :
csv.append([j, c, d])
 j += 1
print('done')
<compute_test_metric>
predict_list = []
predict_list.append(pd.read_csv(".. /input/cellstack/submission-174.csv")[LABELS].values)
 predict_list.append(pd.read_csv(".. /input/cellstack/submission-201.csv")[LABELS].values)
 predict_list.append(pd.read_csv(".. /input/cellstack/submission-231.csv")[LABELS].values )
backtest2(sequences_train, model, given, total_missing )<save_to_csv>
warnings.filterwarnings("ignore")
 print("Rank averaging on ", len(predict_list), " files")
 predictions = np.zeros_like(predict_list[0])
 for predict in predict_list:
for i in range(1):
predictions[:, i] = np.add(predictions[:, i], rankdata(predict[:, i])/predictions.shape[0])
 predictions = predictions /len(predict_list)
 submission = pd.read_csv('.. /input/recursion-cellular-image-classification/sample_submission.csv')
 submission[LABELS] = predictions
submission.to_csv('AggStacker.csv', index=False )
sub = pd.read_csv('.. /input/submission2/submission.csv', header=None,dtype=np.float32)
 sub = pd.DataFrame(sub.values, columns =['ForecastId','ConfirmedCases','Fatalities'])
 sub.ConfirmedCases.astype(np.double)
 sub.Fatalities.astype(np.double)
 sub.ForecastId = sub.ForecastId.astype(np.int)
 sub.to_csv("submission.csv", index=False)
 print('done')
<load_from_csv>
sub_path = ".. /input/cellstack"
all_files = os.listdir(sub_path)
 all_files
train = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv')
 submission = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/submission.csv')
 test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv' )<data_type_conversions>
concat_sub['m_max'] = concat_sub.iloc[:, 1:ncol].max(axis=1)
 concat_sub['m_min'] = concat_sub.iloc[:, 1:ncol].min(axis=1)
 concat_sub['m_median'] = concat_sub.iloc[:, 1:ncol].median(axis=1 )
def use_country(state, country):
if pd.isna(state):
return country
else:
return state
train['Province_State'] = train.apply(lambda x: use_country(x['Province_State'], x['Country_Region']), axis=1)
 test['Province_State'] = test.apply(lambda x: use_country(x['Province_State'], x['Country_Region']), axis=1)
 train['Province_State'].fillna('', inplace=True)
 test['Province_State'].fillna('', inplace=True)
 train['Date'] = pd.to_datetime(train['Date'])
 test['Date'] = pd.to_datetime(test['Date'] )<data_type_conversions>
cutoff_lo = 0.8
cutoff_hi = 0.2
df_out = pd.DataFrame({'ForecastId': [], 'ConfirmedCases': [], 'Fatalities': []})
 for country in set(train['Country_Region']):
country_df = train[train['Country_Region'] == country]
for state in set(country_df['Province_State']):
df = country_df[country_df['Province_State'] == state]
df['Date'] = df['Date'].dt.strftime("%m%d")
 df['Date'] = df['Date'].astype(int)
 test_df= test[(test['Country_Region'] == country)&(test['Province_State'] == state)]
X_Test_CS_Id = test_df['ForecastId']
test_df['Date'] = test_df['Date'].dt.strftime("%m%d")
 test_df['Date'] = test_df['Date'].astype(int)
 X_Train_CS = df[['Country_Region', 'Province_State', 'Date']]
X_Train_CS['Country_Region'] = 0
X_Train_CS['Province_State'] = 0
X_Test_CS = test_df[['Country_Region', 'Province_State', 'Date']]
X_Test_CS['Country_Region'] = 0
X_Test_CS['Province_State'] = 0
y1_Train_CS = df['ConfirmedCases']
y2_Train_CS = df['Fatalities']
model1 = XGBRegressor(n_estimators=1000)
 model1.fit(X_Train_CS, y1_Train_CS)
 y1_pred = model1.predict(X_Test_CS)
 model2 = XGBRegressor(n_estimators=1000)
 model2.fit(X_Train_CS, y2_Train_CS)
 y2_pred = model2.predict(X_Test_CS)
 df = pd.DataFrame({'ForecastId': X_Test_CS_Id, 'ConfirmedCases': y1_pred, 'Fatalities': y2_pred})
 df_out = pd.concat([df_out, df], axis=0 )<save_to_csv>
concat_sub['sirna'] = m_gmean.astype(int)
 concat_sub[['id_code','sirna']].to_csv('stack_mean.csv',
index=False, float_format='%.6f' )
df_out.ForecastId = df_out.ForecastId.astype('int')
 df_out.tail()
df_out.to_csv('submission.csv', index=False )<import_modules>
concat_sub['sirna'] = concat_sub['m_median'].astype(int)
 concat_sub[['id_code','sirna']].to_csv('stack_median.csv',
index=False, float_format='%.6f' )
tqdm.pandas()<define_variables>
concat_sub['sirna'] = np.where(np.all(concat_sub.iloc[:,1:ncol] > cutoff_lo, axis=1), 1,
np.where(np.all(concat_sub.iloc[:,1:ncol] < cutoff_hi, axis=1),
0, concat_sub['m_median']))
concat_sub[['id_code','sirna']].to_csv('stack_pushout_median.csv',
index=False, float_format='%.6f' )
contraction_mapping = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have", "couldn't": "could not",
"didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not",
"he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you", "how'll": "how will", "how's": "how is",
"I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have","I'm": "I am", "I've": "I have", "i'd": "i would",
"i'd've": "i would have", "i'll": "i will", "i'll've": "i will have","i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would",
"it'd've": "it would have", "it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us", "ma'am": "madam", "mayn't": "may not",
"might've": "might have","mightn't": "might not","mightn't've": "might not have", "must've": "must have", "mustn't": "must not", "mustn't've": "must not have",
"needn't": "need not", "needn't've": "need not have","o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not",
"sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would", "she'd've": "she would have", "she'll": "she will", "she'll've": "she will have",
"she's": "she is", "should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have","so's": "so as",
"this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is", "there'd": "there would", "there'd've": "there would have",
"there's": "there is", "here's": "here is","they'd": "they would", "they'd've": "they would have", "they'll": "they will", "they'll've": "they will have",
"they're": "they are", "they've": "they have", "to've": "to have", "wasn't": "was not", "we'd": "we would", "we'd've": "we would have", "we'll": "we will",
"we'll've": "we will have", "we're": "we are", "we've": "we have", "weren't": "were not", "what'll": "what will", "what'll've": "what will have",
"what're": "what are", "what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did", "where's": "where is",
"where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have", "why's": "why is", "why've": "why have",
"will've": "will have", "won't": "will not", "won't've": "will not have", "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have",
"y'all": "you all", "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have","you'd": "you would",
"you'd've": "you would have", "you'll": "you will", "you'll've": "you will have", "you're": "you are", "you've": "you have"}
contraction_patterns = [(r'won't', 'will not'),(r'can't', 'cannot'),(r'i'm', 'i am'),(r'ain't', 'is not'),(r'(\w+)'ll', '\g<1> will'),(r'(\w+)n't', '\g<1> not'), 
(r'(\w+)'ve', '\g<1> have'),(r'(\w+)'s', '\g<1> is'),(r'(\w+)'re', '\g<1> are'),(r'(\w+)'d', '\g<1> would'),(r'&', 'and'),(r'dammit', 'damn it'), 
(r'dont', 'do not'),(r'wont', 'will not')]
punct_mapping = {"‘": "'", "₹": "e", "´": "'", "°": "", "€": "e", "™": "tm", "√": " sqrt ", "×": "x", "²": "2", "—": "-", "–": "-", "’": "'", "_": "-", "`": "'", '“': '"', '”': '"', '“': '"', "£": "e", '∞': 'infinity', 'θ': 'theta', '÷': '/', 'α': 'alpha', '•': '.', 'à': 'a', '−': '-', 'β': 'beta', '∅': '', '³': '3', 'π': 'pi', }
punct = "/-'?!.,
mispell_dict = {'advanatges': 'advantages', 'irrationaol': 'irrational' , 'defferences': 'differences','lamboghini':'lamborghini','hypothical':'hypothetical', 'colour': 'color',
'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor',
'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'qoura' : 'quora', 'sallary': 'salary', 'Whta': 'What',
'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do',
'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', "mastrubating": 'masturbating',
'pennis': 'penis', 'Etherium': 'Ethereum', 'etherium': 'ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota',
'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', "whst": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization',
'demonitization': 'demonetization', 'demonetisation': 'demonetization', 'pokémon': 'pokemon'} 
<string_transform>
concat_sub['m_mean'] = m_gmean.astype(int)
 concat_sub['sirna'] = np.where(np.all(concat_sub.iloc[:,1:ncol] > cutoff_lo, axis=1),
concat_sub['m_max'],
np.where(np.all(concat_sub.iloc[:,1:ncol] < cutoff_hi, axis=1),
concat_sub['m_min'],
concat_sub['m_mean'])).astype(int)
 concat_sub[['id_code','sirna']].to_csv('stack_minmax_mean.csv',
index=False, float_format='%.6f' )
<load_from_csv><EOS> 
concat_sub['sirna'] = np.where(np.all(concat_sub.iloc[:,1:ncol] > cutoff_lo, axis=1),
concat_sub['m_max'],
np.where(np.all(concat_sub.iloc[:,1:ncol] < cutoff_hi, axis=1),
concat_sub['m_min'],
concat_sub['m_median'])).astype(int)
 concat_sub[['id_code','sirna']].to_csv('stack_minmax_median.csv',
index=False, float_format='%.6f' )