Kaggle competition: COVID19 Global Forecasting (Week 2)
from keras.callbacks import Callback, EarlyStopping, ModelCheckpoint
from keras.engine import Layer
from keras.layers import Activation, Add, Bidirectional, Conv1D, Dense, Dropout, Embedding, Flatten
from keras.layers import concatenate, GRU, Input, K, LSTM, MaxPooling1D
from keras.layers import GlobalAveragePooling1D, GlobalMaxPooling1D, SpatialDropout1D
from keras.models import Model
from keras.optimizers import Adam
from keras.preprocessing import text, sequence
from sklearn.metrics import accuracy_score, roc_auc_score, log_loss
from sklearn.model_selection import train_test_split
from keras import initializers, regularizers, constraints, optimizers, layers, callbacks<init_hyperparams>
df_train = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-2/train.csv')
 df_test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-2/test.csv')
 sample_submission = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-2/submission.csv' )
gru_len = 128
Routings = 5
Num_capsule = 10
Dim_capsule = 16
dropout_p = 0.3
rate_drop_dense = 0.3<statistical_test>
df_train_original = df_train
df_test_original = df_test
def squash(x, axis=-1):
s_squared_norm = K.sum(K.square(x), axis, keepdims=True)
 scale = K.sqrt(s_squared_norm + K.epsilon())
 return x / scale<choose_model_class>
df_train = df_train_original.replace(np.nan, '', regex=True)
 df_test = df_test_original.replace(np.nan, '', regex=True )
class Capsule(Layer):
def __init__(self, num_capsule, dim_capsule, routings=3, kernel_size=(9, 1), share_weights=True,
activation='default', **kwargs):
super(Capsule, self ).__init__(**kwargs)
 self.num_capsule = num_capsule
self.dim_capsule = dim_capsule
self.routings = routings
self.kernel_size = kernel_size
self.share_weights = share_weights
if activation == 'default':
self.activation = squash
else:
self.activation = Activation(activation)
 def build(self, input_shape):
super(Capsule, self ).build(input_shape)
 input_dim_capsule = input_shape[-1]
if self.share_weights:
self.W = self.add_weight(name='capsule_kernel',
shape=(1, input_dim_capsule,
self.num_capsule * self.dim_capsule),
initializer='glorot_uniform',
trainable=True)
 else:
input_num_capsule = input_shape[-2]
self.W = self.add_weight(name='capsule_kernel',
shape=(input_num_capsule,
input_dim_capsule,
self.num_capsule * self.dim_capsule),
initializer='glorot_uniform',
trainable=True)
 def call(self, u_vecs):
if self.share_weights:
u_hat_vecs = K.conv1d(u_vecs, self.W)
 else:
u_hat_vecs = K.local_conv1d(u_vecs, self.W, [1], [1])
 batch_size = K.shape(u_vecs)[0]
input_num_capsule = K.shape(u_vecs)[1]
u_hat_vecs = K.reshape(u_hat_vecs,(batch_size, input_num_capsule,
self.num_capsule, self.dim_capsule))
u_hat_vecs = K.permute_dimensions(u_hat_vecs,(0, 2, 1, 3))
b = K.zeros_like(u_hat_vecs[:, :, :, 0])
 for i in range(self.routings):
b = K.permute_dimensions(b,(0, 2, 1))
c = K.softmax(b)
 c = K.permute_dimensions(c,(0, 2, 1))
b = K.permute_dimensions(b,(0, 2, 1))
outputs = self.activation(K.batch_dot(c, u_hat_vecs, [2, 2]))
if i < self.routings - 1:
b = K.batch_dot(outputs, u_hat_vecs, [2, 3])
 return outputs
def compute_output_shape(self, input_shape):
return(None, self.num_capsule, self.dim_capsule )<choose_model_class>
df_train['location'] = df_train[['Country_Region', 'Province_State']].apply(lambda x: '-'.join(x), axis=1)
 df_test['location'] = df_test[['Country_Region', 'Province_State']].apply(lambda x: '-'.join(x), axis=1 )
def get_model(embedding_matrix, sequence_length, dropout_rate, recurrent_units, dense_size):
input1 = Input(shape=(sequence_length,))
embed_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],
weights=[embedding_matrix], trainable=False )(input1)
 embed_layer = SpatialDropout1D(rate_drop_dense )(embed_layer)
 x = Bidirectional(
 GRU(gru_len, activation='relu', dropout=dropout_p, recurrent_dropout=dropout_p, return_sequences=True))(
 embed_layer)
 capsule = Capsule(num_capsule=Num_capsule, dim_capsule=Dim_capsule, routings=Routings,
share_weights=True )(x)
 capsule = Flatten()(capsule)
 capsule = Dropout(dropout_p )(capsule)
 output = Dense(1, activation='sigmoid' )(capsule)
 model = Model(inputs=input1, outputs=output)
 model.compile(
 loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
 return model<train_model>
df_train[df_train['ConfirmedCases'] > 0]
def _train_model(model, batch_size, train_x, train_y, val_x, val_y):
num_labels = train_y.shape[1]
patience = 5
best_loss = -1
best_weights = None
best_epoch = 0
current_epoch = 0
while True:
model.fit(train_x, train_y, batch_size=batch_size, epochs=1)
 y_pred = model.predict(val_x, batch_size=batch_size)
 total_loss = 0
for j in range(num_labels):
loss = log_loss(val_y[:, j], y_pred[:, j])
 total_loss += loss
total_loss /= num_labels
print("Epoch {0} loss {1} best_loss {2}".format(current_epoch, total_loss, best_loss))
current_epoch += 1
if total_loss < best_loss or best_loss == -1:
best_loss = total_loss
best_weights = model.get_weights()
best_epoch = current_epoch
else:
if current_epoch - best_epoch == patience:
break
model.set_weights(best_weights)
 return model<train_model>
groups_train = df_train.groupby(['Country_Region', 'Province_State'])
 print(len(groups_train))
def train_folds(X, y, X_test, fold_count, batch_size, get_model_func):
print("="*75)
 fold_size = len(X)// fold_count
models = []
result_path = "predictions"
if not os.path.exists(result_path):
os.mkdir(result_path)
 for fold_id in range(0, fold_count):
fold_start = fold_size * fold_id
fold_end = fold_start + fold_size
if fold_id == fold_size - 1:
fold_end = len(X)
 train_x = np.concatenate([X[:fold_start], X[fold_end:]])
 train_y = np.concatenate([y[:fold_start], y[fold_end:]])
 val_x = np.array(X[fold_start:fold_end])
 val_y = np.array(y[fold_start:fold_end])
 model = _train_model(get_model_func() , batch_size, train_x, train_y, val_x, val_y)
 train_predicts_path = os.path.join(result_path, "train_predicts{0}.npy".format(fold_id))
test_predicts_path = os.path.join(result_path, "test_predicts{0}.npy".format(fold_id))
train_predicts = model.predict(X, batch_size=512, verbose=1)
 test_predicts = model.predict(X_test, batch_size=512, verbose=1)
 np.save(train_predicts_path, train_predicts)
 np.save(test_predicts_path, test_predicts)
 return models<define_variables>
train_file_path = ".. /input/donorschooseorg-preprocessed-data/train_small.csv"
test_file_path = ".. /input/donorschooseorg-preprocessed-data/test_small.csv"
embedding_path = ".. /input/donorschooseorg-preprocessed-data/embeddings_small.vec"
batch_size = 128
recurrent_units = 16
dropout_rate = 0.3
dense_size = 8
sentences_length = 10
fold_count = 2<define_variables>
min_date_sorted = min_date.sort_values()
UNKNOWN_WORD = "_UNK_"
END_WORD = "_END_"
NAN_WORD = "_NAN_"
CLASSES = ["project_is_approved"]<load_from_csv>
for x,y in zip(min_date_sorted.index, min_date_sorted):
print(x,y )
print("Loading data...")
 train_data = pd.read_csv(train_file_path)
 test_data = pd.read_csv(test_file_path)
 list_sentences_train = train_data["application_text"].fillna(NAN_WORD ).values
list_sentences_test = test_data["application_text"].fillna(NAN_WORD ).values
y_train = train_data[CLASSES].values<string_transform>
list(df_train[df_train['Country_Region'] == 'China']['Province_State'] )
print("Tokenizing sentences in train set...")
 tokenized_sentences_train, words_dict = tokenize_sentences(list_sentences_train, {})
 print("Tokenizing sentences in test set...")
 tokenized_sentences_test, words_dict = tokenize_sentences(list_sentences_test, words_dict )<load_pretrained>
df_train[(df_train['Country_Region'] == 'Pakistan')]
words_dict[UNKNOWN_WORD] = len(words_dict)
 print("Loading embeddings...")
 embedding_list, embedding_word_dict = read_embedding_list(embedding_path)
 embedding_size = len(embedding_list[0] )<categorify>
df_train[(df_train['Country_Region'] == 'US')&(df_train['Province_State'] == 'Washington')]
print("Preparing data...")
 embedding_list, embedding_word_dict = clear_embedding_list(embedding_list, embedding_word_dict, words_dict)
 embedding_word_dict[UNKNOWN_WORD] = len(embedding_word_dict)
 embedding_list.append([0.] * embedding_size)
 embedding_word_dict[END_WORD] = len(embedding_word_dict)
 embedding_list.append([-1.] * embedding_size)
 embedding_matrix = np.array(embedding_list)
 id_to_word = dict(( id, word)for word, id in words_dict.items())
 train_list_of_token_ids = convert_tokens_to_ids(
 tokenized_sentences_train,
id_to_word,
embedding_word_dict,
sentences_length)
 test_list_of_token_ids = convert_tokens_to_ids(
 tokenized_sentences_test,
id_to_word,
embedding_word_dict,
sentences_length)
 X_train = np.array(train_list_of_token_ids)
 X_test = np.array(test_list_of_token_ids )<load_pretrained>
index = 0
for x,y in zip(min_date_sorted.index, min_date_sorted):
print(index, x, y)
 index = index + 1
get_model_func = lambda: get_model(
 embedding_matrix,
sentences_length,
dropout_rate,
recurrent_units,
dense_size )<drop_column>
import matplotlib.pyplot as plt
del train_data, test_data, list_sentences_train, list_sentences_test
del tokenized_sentences_train, tokenized_sentences_test, words_dict
del embedding_list, embedding_word_dict
del train_list_of_token_ids, test_list_of_token_ids
gc.collect() ;<train_model>
index = 34
print("Starting to train models...")
 models = train_folds(X_train, y_train, X_test, fold_count, batch_size, get_model_func )<save_to_csv>
record = df_train[(df_train['Country_Region'] == min_date_sorted.index[index][0])&(df_train['Province_State'] == min_date_sorted.index[index][1])]
LABELS = ["project_is_approved"]
base = ".. /input/donorschooseorg-application-screening-predictions/predictions/predictions/"
predict_list = []
for j in range(10):
predict_list.append(np.load(base + "predictions_001/test_predicts%d.npy"%j))
print("Rank averaging on ", len(predict_list), " files")
 predcitions = np.zeros_like(predict_list[0])
 for predict in predict_list:
predcitions = np.add(predcitions.flatten() , rankdata(predict)/predcitions.shape[0])
 predcitions /= len(predict_list)
 submission = pd.read_csv('.. /input/donorschoose-application-screening/sample_submission.csv')
 submission[LABELS] = predcitions
submission.to_csv('submission.csv', index=False )<load_from_csv>
x = np.linspace(0, 10, num = 40)
 y = 3.45 * np.sin(1.334 * x)+ np.random.normal(size = 40)
 def test(x, a, b):
return a * np.sin(b * x)
 param, param_cov = curve_fit(test, x, y )
nrows = None
data = pd.read_csv('.. /input/train.csv', nrows=nrows ).drop('teacher_id', axis=1)
 resource_data = pd.read_csv('.. /input/resources.csv')
 y = data['project_is_approved']
X = data.drop('project_is_approved', axis=1 )<load_from_csv>
record = df_train[(df_train['Country_Region'] == min_date_sorted.index[index][0])&(df_train['Province_State'] == min_date_sorted.index[index][1])]
record = record[record['ConfirmedCases'] > 0]
base_date_object = datetime.strptime('2020-01-22', "%Y-%m-%d" ).date()
record['days'] = [(datetime.strptime(date, "%Y-%m-%d" ).date() - base_date_object ).days + 1 for date in record['Date']]
resource_data = pd.read_csv('.. /input/resources.csv')
 resource_data['total_price'] = resource_data['quantity'] * resource_data['price']
resource_data['description'] = resource_data['description'] + ' '
resource_data = pd.DataFrame(resource_data.groupby('id' ).apply(lambda x: x.sum()))
resource_data = resource_data.reset_index()
resource_data = resource_data.rename(columns={'level_1':'attribute', 0:'value'})
 resource_data = resource_data.pivot(index='id', columns='attribute', values='value' ).drop(columns=['id','price'] ).reset_index()<feature_engineering>
record2 = record[record['Fatalities'] > 0]
x = record['days'].values
x2 = record2['days'].values
y1 = record['ConfirmedCases'].values
y2 = record2['Fatalities'].values
class StemmedTfidfVectorizer(TfidfVectorizer):
def __init__(self, stemmer, stop_words, max_features):
super(StemmedTfidfVectorizer, self ).__init__(stop_words=stop_words, max_features=max_features)
 self.stemmer = stemmer
def build_analyzer(self):
analyzer = super(StemmedTfidfVectorizer, self ).build_analyzer()
return lambda doc:(self.stemmer.stem(w)for w in analyzer(doc))<merge>
def gaussian(x, amp, cen, wid):
return amp * exp(-(x-cen)**2 / wid)
 def test(x, a, b, c):
return a*1/(1+exp(-b*(x-c)))
 def test_linear(x, a, b, c, d, e, f):
return a*1/(1+exp(-b*(x-c)))+ d*log(1+exp(x-e)) - f
def custom(x, a, b , c, d, e, f, g):
return a*1/(1+exp(-(x-b)/c)) *(d*1/(1+exp(-(x-e)/f)) + g )
def join_resource_features(df, resource_data):
joined_df = df.merge(resource_data, on='id', how='left')
 return joined_df
def combine_essays(row):

 if row['project_submitted_datetime'] < '2016-05-17':
row['project_essay_1'] = row['project_essay_1'] \
+ ' ' + row['project_essay_2']
row['project_essay_2'] = row['project_essay_3'] \
+ ' ' + row['project_essay_4']
return row
def preprocess_text(X_train, X_test):

 X_train.apply(lambda row: combine_essays(row), axis=1)
 X_test.apply(lambda row: combine_essays(row), axis=1)
 X_train['description'] = X_train['description'].fillna(' ')
 X_test['description'] = X_test['description'].fillna(' ')
 X_train['description'] = X_train['description'].astype(str)
 X_test['description'] = X_test['description'].astype(str)
 X_train = X_train.drop(columns=['project_essay_3','project_essay_4'], axis=1)
 X_test = X_test.drop(columns=['project_essay_3','project_essay_4'], axis=1)
 stemmer = EnglishStemmer()
count_vec_title = CountVectorizer(stop_words='english', max_features=50)
 count_vec_title.fit(X_train['project_title'])
 tfidf_vec_title = StemmedTfidfVectorizer(stemmer=stemmer, stop_words='english', max_features=500)
 tfidf_vec_title.fit(X_train['project_title'])
 count_vec_essay1 = CountVectorizer(stop_words='english', max_features=50)
 count_vec_essay1.fit(X_train['project_essay_1'])
 tfidf_vec_essay1 = StemmedTfidfVectorizer(stemmer=stemmer, stop_words='english', max_features=500)
 tfidf_vec_essay1.fit(X_train['project_essay_1'])
 count_vec_essay2 = CountVectorizer(stop_words='english', max_features=50)
 count_vec_essay2.fit(X_train['project_essay_2'])
 tfidf_vec_essay2 = StemmedTfidfVectorizer(stemmer=stemmer, stop_words='english', max_features=500)
 tfidf_vec_essay2.fit(X_train['project_essay_2'])
 count_vec_resource = CountVectorizer(stop_words='english', max_features=50)
 count_vec_resource.fit(X_train['project_resource_summary'])
 tfidf_vec_resource = StemmedTfidfVectorizer(stemmer=stemmer, stop_words='english', max_features=500)
 tfidf_vec_resource.fit(X_train['project_resource_summary'])
 count_vec_resource_desc = CountVectorizer(stop_words='english', max_features=50)
 count_vec_resource_desc.fit(X_train['description'])
 tfidf_vec_resource_desc = StemmedTfidfVectorizer(stemmer=stemmer, stop_words='english', max_features=500)
 tfidf_vec_resource_desc.fit(X_train['description'])
 return(X_train, X_test,
count_vec_title, tfidf_vec_title,
count_vec_essay1, tfidf_vec_essay1,
count_vec_essay2, tfidf_vec_essay2,
count_vec_resource, tfidf_vec_resource,
count_vec_resource_desc, tfidf_vec_resource_desc)
 def process_features(df,
count_vec_title, tfidf_vec_title,
count_vec_essay1, tfidf_vec_essay1,
count_vec_essay2, tfidf_vec_essay2,
count_vec_resource, tfidf_vec_resource,
count_vec_resource_desc, tfidf_vec_resource_desc):

 dropped_columns = ['project_submitted_datetime','project_subject_categories','project_subject_subcategories',
'project_title','project_essay_1','project_essay_2','project_resource_summary','description']
ids = df['id']
df = df.drop('id', axis=1)
 df['quantity'] = df['quantity'].astype(float)
 df['total_price'] = df['total_price'].astype(float)
 df = pd.get_dummies(df, columns=['teacher_prefix'])
 df = pd.get_dummies(df, columns=['school_state'])
 df = pd.get_dummies(df, columns=['project_grade_category'])
 subjects = ['Applied Learning', 'Health & Sports', 'History & Civics', 'Literacy & Language', 'Math & Science', 'Music & The Arts', 'Special Needs', 'Warmth, Care & Hunger']
subject_subcategories = { 'Applied Learning': ['Character Education', 'College & Career Prep', 'Community Service', 'Early Development', 'Extracurricular', 'Other', 'Parent Involvement'],
'Health & Sports': ['Gym & Fitness', 'Health & Wellness', 'Nutrition Education', 'Team Sports'],
'History & Civics': ['Civics & Government', 'Economics', 'Financial Literacy', 'History & Geography', 'Social Sciences'],
'Literacy & Language': ['ESL', 'Foreign Languages', 'Literacy', 'Literature & Writing'],
'Math & Science': ['Applied Sciences', 'Environmental Science', 'Health & Life Science', 'Mathematics'],
'Music & The Arts': ['Music', 'Performing Arts', 'Visual Arts'],
'Special Needs': [],
'Warmth, Care & Hunger': [] }
for subject in subjects:
df['Subject: ' + subject] = df['project_subject_categories'].str.contains(subject ).astype(int)
 for subcategory in subject_subcategories[subject]:
df['Subcategory: ' + subcategory] = df['project_subject_subcategories'].str.contains(subcategory ).astype(int)
 df['project_submitted_datetime'] = pd.to_datetime(df['project_submitted_datetime'])
 df['submit_year'] = df['project_submitted_datetime'].dt.year
df['submit_month'] = df['project_submitted_datetime'].dt.month
df['submit_day'] = df['project_submitted_datetime'].dt.day
df['submit_dayofweek'] = df['project_submitted_datetime'].dt.weekday_name
df['submit_hour'] = df['project_submitted_datetime'].dt.hour
df = pd.get_dummies(df, columns=['submit_dayofweek'])
 pattern = r'\\([a-zA-Z]|")'
df['project_title'] = df['project_title'].str.replace(pattern,'')
 df['title_polarity'] = df['project_title'].apply(lambda x: TextBlob(x ).sentiment.polarity)
 df['project1_polarity'] = df['project_essay_1'].apply(lambda x: TextBlob(x ).sentiment.polarity)
 df['project2_polarity'] = df['project_essay_2'].apply(lambda x: TextBlob(x ).sentiment.polarity)
 df['resource_polarity'] = df['project_resource_summary'].apply(lambda x: TextBlob(x ).sentiment.polarity)
 title_dtm = count_vec_title.transform(df['project_title'])
 project1_dtm = count_vec_essay1.transform(df['project_essay_1'])
 project2_dtm = count_vec_essay2.transform(df['project_essay_2'])
 resource_dtm = count_vec_resource.transform(df['project_resource_summary'])
 resource_desc_dtm = count_vec_resource_desc.transform(df['description'])
 title_tfidf_dtm = tfidf_vec_title.transform(df['project_title'])
 project1_tfidf_dtm = tfidf_vec_essay1.transform(df['project_essay_1'])
 project2_tfidf_dtm = tfidf_vec_essay2.transform(df['project_essay_2'])
 resource_tfidf_dtm = tfidf_vec_resource.transform(df['project_resource_summary'])
 resource_desc_tfidf_dtm = tfidf_vec_resource_desc.transform(df['description'])
 df['project_title'] = df['project_title'].apply(str.lower)
 df['project_title'] = df['project_title'].apply(word_tokenize)
 df['has_exclamation'] = df['project_title'].apply(lambda x: '!' in x ).astype(int)
 df['title_token_count'] = df['project_title'].apply(len)
 df = df.drop(dropped_columns, axis=1)
 processed_array = hstack(( title_tfidf_dtm,
project1_tfidf_dtm, project2_tfidf_dtm,
resource_tfidf_dtm, resource_desc_dtm, resource_desc_tfidf_dtm, np.array(df)[:,:]))
return(ids, processed_array )<load_from_csv>
y_max_ = y1[-1]
y1_prime = np.diff(y1)
 y1_prime2 = np.diff(y1_prime)
 if len(y1)>0 and len(y1_prime)> 0 and len(y1_prime2)> 0:
max_slope_index = len(y1_prime)- 1 - list(y1_prime)[::-1].index(max(y1_prime))
max_slope_range =(max_slope_index+1)/len(y1_prime)
 y_max_ = y1[-1]
if max_slope_range < 0.75:
if y1_prime[max_slope_index] > 0 and max_slope_range < 0.75 and(((y1_prime[max_slope_index] - max(y1_prime[-2:])) /y1_prime[max_slope_index])< 0.5):
y_max_ = y1[-1]
pass
else:
y_max_ = y1[max_slope_index + 1]
pass
else:
y_max_ = y1[-1]
X_train = X
X_test = pd.read_csv('.. /input/test.csv', nrows=nrows ).drop('teacher_id', axis=1 )<merge>
param
X_train = join_resource_features(X_train, resource_data)
 X_test = join_resource_features(X_test, resource_data)
(X_train_preprocessed, X_test_preprocessed,
count_vec_title, tfidf_vec_title,
count_vec_essay1, tfidf_vec_essay1,
count_vec_essay2, tfidf_vec_essay2,
count_vec_resource, tfidf_vec_resource,
count_vec_resource_desc, tfidf_vec_resource_desc)= preprocess_text(X_train, X_test)
(X_train_ids, X_train)= process_features(X_train_preprocessed,
count_vec_title, tfidf_vec_title,
count_vec_essay1, tfidf_vec_essay1,
count_vec_essay2, tfidf_vec_essay2,
count_vec_resource, tfidf_vec_resource,
count_vec_resource_desc, tfidf_vec_resource_desc)
(X_test_ids, X_test)= process_features(X_test_preprocessed,
count_vec_title, tfidf_vec_title,
count_vec_essay1, tfidf_vec_essay1,
count_vec_essay2, tfidf_vec_essay2,
count_vec_resource, tfidf_vec_resource,
count_vec_resource_desc, tfidf_vec_resource_desc )<save_to_csv>
y1_pred = test(x,param[0], param[1], param[2])
 base_x = range(61,100,1)
 y1_pred_test = test(base_x, param[0], param[1], param[2])
xgb_model = XGBClassifier(max_depth=2, n_estimators=1000, learning_rate=0.20, objective='binary:logistic')
 xgb_model.fit(X_train, y)
 predictions = xgb_model.predict_proba(X_test)
 output_df = pd.DataFrame(columns=['id','project_is_approved'])
 output_df['id'] = X_test_ids
output_df['project_is_approved'] = predictions[:,1]
output_df.to_csv('submission.csv',index=False )<define_variables>
( x2[-1] - x2[0])/2 + x2[0]
quick_run = False
max_features = 80000
embed_size = 300
pj_repeat = 3
rs_repeat = 1
dpcnn_folds = 5
batch_size = 64
epochs = 5
project_maxlen = 210
resouse_max_len = 30
maxlen = project_maxlen + resouse_max_len
if quick_run == True:
max_features = 1000
epochs = 2
EMBEDDING_FILE = '.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec'<set_options>
y2_pred = test(x2,param2[0], param2[1], param2[2] )
warnings.filterwarnings('ignore' )<load_from_csv>
base_x = range(61,100,1)
 print(len(base_x))
base_y1 = test(base_x,param[0], param[1], param[2])
 base_y2 = test(base_x,param2[0], param2[1], param2[2] )
train_df = pd.read_csv('.. /input/donorschoose-application-screening/train.csv')
 test_df = pd.read_csv('.. /input/donorschoose-application-screening/test.csv')
 resouse_df = pd.read_csv('.. /input/donorschoose-application-screening/resources.csv' )<create_dataframe>
day_index_pred = 0
diff1_list = []
diff2_list = []
for day in base_x:
if day in x:
day_index = np.where(x == day)
 diff1 = y1[day_index] - base_y1[day_index_pred]
diff1_list.append(diff1)
 if day in x2:
day_index = np.where(x2 == day)
 diff2 = y2[day_index] - base_y2[day_index_pred]
diff2_list.append(diff2)
 day_index_pred = day_index_pred + 1
diff1_mean = np.max(diff1_list)
 diff2_mean = np.max(diff2_list)
 if np.isnan(diff1_mean):
pass
else:
base_y1_mod = list(np.array(base_y1)+ diff1_mean)
 if np.isnan(diff2_mean):
pass
else:
base_y2_mod = list(np.array(base_y2)+ diff2_mean)
 base_y1_pred = [int(n)for n in base_y1_mod]
base_y2_pred = [int(m)for m in base_y2_mod]
%%time
resouse_df['description'].fillna('', inplace=True)
 res_nums = pd.DataFrame(resouse_df[['id', 'price']].groupby('id' ).price.agg(['count',
'sum',
'min',
'max',
'mean',
'std',
lambda x: len(np.unique(x)) ,])).reset_index()
res_nums = res_nums.rename(columns={'count': 'res_count',
'sum': 'res_sum',
'min': 'res_min',
'max': 'res_max',
'mean': 'res_mean',
'std': 'res_std',
'<lambda>': 'res_unique' })
 res_descp = resouse_df[['id', 'description']].groupby('id' ).description.agg([ lambda x: ' '.join(x)] ).reset_index().rename(columns={'<lambda>':'res_description'})
 resouse_df = res_nums.merge(res_descp, on='id', how='left')
 train_df = train_df.merge(resouse_df, on='id', how='left')
 test_df = test_df.merge(resouse_df, on='id', how='left')
 del res_nums
del res_descp
del resouse_df
gc.collect()<split>
test_groups = df_test.groupby(['Country_Region', 'Province_State'] )
if quick_run == True:
train_df = train_df[:10000]
test_df = test_df[:100]<prepare_x_and_y>
index = 0
for key_,_ in zip(min_date_sorted.index, min_date_sorted):
record = df_train[(df_train['Country_Region'] == key_[0])&(df_train['Province_State'] == key_[1])]
record['days'] = [(datetime.strptime(date, "%Y-%m-%d" ).date() - base_date_object ).days + 1 for date in record['Date']]
x = record['days']
y1 = record['ConfirmedCases']
y2 = record['Fatalities']
y1_prime = np.diff(y1)
 stage0 = False
stage1 = False
stage2 = False
stage3 = False
count1 = 0
count2 = 0
for start in range(len(y1_prime)-3):
if sum(y1_prime[start:start+3])<=12:
count1 = count1 + 1
count2 = 0
else:
count2 = count2 + 1
count1 = 0
if not stage0 and count2 == 0 and count1 > 2:
stage0 = True
count1 = 0
if not stage1 and count1 == 0 and count2 > 5:
stage0 = True
stage1 = True
count2 = 0
if stage1 and count2 == 0 and count1 > 3:
stage2 = True
count1 = 0
if stage2 and count1 == 0 and count2 > 2:
stage3 = True
count2 = 0
if stage3:
print(index, key_)
 print(y1_prime)
 plt.plot(x, y1, label = "Confirmed Cases")
 plt.xlabel('Date')
 plt.ylabel('Label')
 plt.title(str(key_[0])+ " " + str(key_[1])+ ' - Confirmed Cases')
 plt.show()
index = index + 1
train_target = train_df['project_is_approved'].values
gc.collect()<feature_engineering>
total_confirmed = 0
total_fatalities = 0
rate = []
max_y1 = []
max_y2 = []
details = []
for index, start_date in zip(min_date_sorted.index, min_date_sorted):
print(index, start_date)
 record = df_train[(df_train['Country_Region'] == index[0])&(df_train['Province_State'] == index[1])]
if len(record[record['ConfirmedCases'] > 0])!= 0:
record = record[record['ConfirmedCases'] > 0]
record2 = record
if len(record[record['Fatalities'] > 0])!= 0:
record2 = record[record['Fatalities'] > 0]
y1 = record['ConfirmedCases'].values
y2 = record2['Fatalities'].values
b = -1
bad_index = 0
mod_count = 0
y1_copy = list(y1)
 for a in y1:
if a < b:
y1[bad_index] = b
mod_count = mod_count + 1
else:
b = a
bad_index = bad_index + 1
b = -1
bad_index = 0
mod_count = 0
y2_copy = list(y2)
 for a in y2:
if a < b:
y2[bad_index] = b
mod_count = mod_count + 1
else:
b = a
bad_index = bad_index + 1
y1_prime = np.diff(y1)
 y1_prime2 = np.diff(y1_prime)
 y_max_ = y1[-1]*2 + 1500
if len(y1)>0 and len(y1_prime)> 0 and len(y1_prime2)> 0:
max_slope_index = len(y1_prime)- 1 - list(y1_prime)[::-1].index(max(y1_prime))
max_slope_range =(max_slope_index+1)/len(y1_prime)
 if max_slope_range < 0.75:
if y1_prime[max_slope_index] > 0 and max_slope_range < 0.5 and(((y1_prime[max_slope_index] - max(y1_prime[-2:])) /y1_prime[max_slope_index])< 0.5):
y_max_ = y1[-1]*2 + 1500
pass
else:
y_max_ = y1[max_slope_index + 1]*2 + 1500
pass
else:
y_max_ = y1[-1]*2 + 1500
ratio = 0
if y2[-1] > 0:
ratio = y1[-1]/y2[-1]
else:
ratio = y1[-1]
max_y1.append(y1[-1])
 max_y2.append(y2[-1])
 rate.append(ratio)
 details.append(" ".join([str(x)for x in [y1[-1], " ------- ", y2[-1], " ---- ", ratio, " --------------- ", record['Date'].values[-1], " ---- ", index, "----", list(min_date_sorted.index ).index(index)]]))
total_confirmed = total_confirmed + y1[-1]
total_fatalities = total_fatalities + y2[-1]
print(total_confirmed/total_fatalities )
essay_cols = ['project_essay_1', 'project_essay_2','project_essay_3', 'project_essay_4']
essay_length_cols = [item+'_len' for item in essay_cols]
def count_essay_length(df):
for col in essay_cols:
df[col] = df[col].fillna('unknown')
 df[col+'_len'] = df[col].apply(len)
 return df
train_df = count_essay_length(train_df)
 test_df = count_essay_length(test_df)
 train_df['project_essay'] = ''
test_df['project_essay'] = ''
for col in essay_cols:
train_df['project_essay'] += train_df[col] + ' '
test_df['project_essay'] += test_df[col] + ' '
train_df[['project_essay']].head()<feature_engineering>
for a1, a2, b, c in zip(max_y1, max_y2, rate, details):
print(c )
time_cols = ['sub_year', 'sub_month', 'sub_day', 'sub_hour', 'sub_dayofweek', 'sub_dayofyear']
def time_stamp_features(df):
time_df = pd.to_datetime(df['project_submitted_datetime'])
 df['sub_year'] = time_df.apply(lambda x: x.year)
 df['sub_month'] = time_df.apply(lambda x: x.month)
 df['sub_day'] = time_df.apply(lambda x: x.day)
 df['sub_hour'] = time_df.apply(lambda x: x.hour)
 df['sub_dayofweek'] = time_df.apply(lambda x: x.dayofweek)
 df['sub_dayofyear'] = time_df.apply(lambda x: x.dayofyear)
 return df
train_df = time_stamp_features(train_df)
 test_df = time_stamp_features(test_df )<categorify>
for a1, a2, b, c in zip(max_y1, max_y2, rate, details):
if(a1 < 100 and a2 < 4 and b < avg):
print(c)
 pass
else:
pass
str_cols = ['teacher_id', 'teacher_prefix', 'school_state',
'project_submitted_datetime', 'project_grade_category',
'project_subject_categories', 'project_subject_subcategories',
'project_title', 'project_resource_summary','res_description', 'project_essay']
num_cols = ['teacher_number_of_previously_posted_projects',
'res_count', 'res_sum', 'res_min', 'res_max', 'res_mean', 'res_std', 'res_unique'] + essay_length_cols
train_df[str_cols] =train_df[str_cols].fillna('unknown')
 train_df[num_cols] = train_df[num_cols].fillna(0)
 test_df[str_cols] =test_df[str_cols].fillna('unknown')
 test_df[num_cols] = test_df[num_cols].fillna(0)
 for col in str_cols:
train_df[col] = train_df[col].str.lower()
test_df[col] = test_df[col].str.lower()<normalization>
df = pd.DataFrame(columns = ['ForecastId','ConfirmedCases','Fatalities'])
 df_hr = pd.DataFrame(columns = ['ForecastId', 'Country_Region', 'Province_State', 'Days', 'ConfirmedCases','Fatalities','Date'])
std_scaler = MinMaxScaler()
train_none_text_features = std_scaler.fit_transform(train_df[num_cols].values)
 test_none_text_features = std_scaler.transform(test_df[num_cols].values)
 train_df = train_df.drop(num_cols, axis=1)
 test_df = test_df.drop(num_cols, axis=1)
 del std_scaler
gc.collect()<drop_column>
public_start_date = '2020-03-19'
public_end_date = '2020-04-01'
count = 0
for index, start_date in zip(min_date_sorted.index, min_date_sorted):
print(list(min_date_sorted.index ).index(index), index, start_date)
 record = df_train[(df_train['Country_Region'] == index[0])&(df_train['Province_State'] == index[1])]
if len(record[record['ConfirmedCases'] > 0])== 0:
pass
else:
record = record[record['ConfirmedCases'] > 0]
base_date_object = datetime.strptime(start_date, "%Y-%m-%d" ).date()
public_start_date_object = datetime.strptime(public_start_date, "%Y-%m-%d" ).date()
public_end_date_object = datetime.strptime(public_end_date, "%Y-%m-%d" ).date()
record['days'] = [(datetime.strptime(date, "%Y-%m-%d" ).date() - base_date_object ).days + 1 for date in record['Date']]
public_start_day =(public_start_date_object - base_date_object ).days + 1
public_end_day =(public_end_date_object - base_date_object ).days + 1
if len(record[record['days'] < public_start_day])> 0:
record = record[record['days'] < public_start_day]
record2 = record
if len(record[record['Fatalities'] > 0])!= 0:
record2 = record[record['Fatalities'] > 0]
x = record['days'].values
x2 = record2['days'].values
y1 = record['ConfirmedCases'].values
y2 = record2['Fatalities'].values
b = -1
bad_index = 0
mod_count = 0
for a in y1:
if a < b:
y1[bad_index] = b
mod_count = mod_count + 1
else:
b = a
bad_index = bad_index + 1
if mod_count > 0:
print("*****************")
 print(list(min_date_sorted.index ).index(index), index)
 print(mod_count)
 print(y1)
 print("*****************")
 b = -1
bad_index = 0
mod_count = 0
for a in y2:
if a < b:
y2[bad_index] = b
mod_count = mod_count + 1
else:
b = a
bad_index = bad_index + 1
if mod_count > 0:
print("*****************")
 print(list(min_date_sorted.index ).index(index), index)
 print(mod_count)
 print(y2)
 print("*****************")
 if len(y1)> 0:
y_max_ = y1[-1]
y1_prime = np.diff(y1)
 y1_prime2 = np.diff(y1_prime)
 if len(y1)>0 and len(y1_prime)> 0 and len(y1_prime2)> 0:
max_slope_index = len(y1_prime)- 1 - list(y1_prime)[::-1].index(max(y1_prime))
max_slope_range =(max_slope_index+1)/len(y1_prime)
 y_max_ = y1[-1]
if max_slope_range < 0.75:
if y1_prime[max_slope_index] > 0 and max_slope_range < 0.75 and(((y1_prime[max_slope_index] - max(y1_prime[-2:])) /y1_prime[max_slope_index])< 0.5):
y_max_ = y1[-1]
pass
else:
y_max_ = y1[max_slope_index + 1]
pass
else:
y_max_ = y1[-1]
else:
y_max_ = 0
stage0 = False
stage1 = False
stage2 = False
stage3 = False
count1 = 0
count2 = 0
for start in range(len(y1_prime)-3):
if sum(y1_prime[start:start+3])<=12:
count1 = count1 + 1
count2 = 0
else:
count2 = count2 + 1
count1 = 0
if not stage0 and count2 == 0 and count1 > 2:
stage0 = True
count1 = 0
if not stage1 and count1 == 0 and count2 > 5:
stage0 = True
stage1 = True
count2 = 0
if stage1 and count2 == 0 and count1 > 3:
stage2 = True
count1 = 0
if stage2 and count1 == 0 and count2 > 2:
stage3 = True
count2 = 0
if stage3:
param, param_cov = curve_fit(custom, np.array(x), np.array(y1), maxfev = 100000, bounds=([1, 0, 1, 1, 30, 1, 1], [2, 60, 8, 200, 90, 8, 1400]))
y1_pred = custom(x, param[0], param[1], param[2], param[3], param[4], param[5], param[6])
 elif index[0] == 'Korea, South':
param, param_cov = curve_fit(test_linear, np.array(x), np.array(y1), [y_max_, 0.5,(x[-1] - x[0])/2 + x[0], 50, 45, 0], maxfev = 100000, bounds=([y_max_/2, 0.1, 0, 1, 30, -100],[y_max_*5 + 1500, 1, 150, 100, 100, 1000]))
y1_pred = test_linear(x, param[0], param[1], param[2], param[3], param[4], param[5])
 elif index[0] in ['US', 'Spain', 'Germany', 'France', 'Iran', 'United Kingdom']:
param, param_cov = curve_fit(test, np.array(x), np.array(y1), [y_max_*7, 0.5,(x[-1] - x[0])/2 + x[0]], maxfev = 100000, bounds=([y_max_*5, 0.1, 0],[y_max_*10 + 1500, 1, 150]))
y1_pred = test(x, param[0], param[1], param[2])
 elif index[0] == 'China':
param, param_cov = curve_fit(test, np.array(x), np.array(y1), [y_max_, 0.5,(x[-1] - x[0])/2 + x[0]], maxfev = 100000, bounds=([y_max_/2, 0.1, 0],[y_max_*5 + 1500, 1, 150]))
y1_pred = test(x, param[0], param[1], param[2])
 else:
param, param_cov = curve_fit(test, np.array(x), np.array(y1), [y_max_*5, 0.5,(x[-1] - x[0])/2 + x[0]], maxfev = 100000, bounds=([y_max_*4, 0.1, 0],[y_max_*8 + 1500, 1, 150]))
y1_pred = test(x, param[0], param[1], param[2])
 param2, param_cov2 = curve_fit(test, np.array(x2), np.array(y2), [y2[-1]/2, 0.5,(x2[-1] - x2[0])/2 + x2[0] - 3], maxfev = 100000, bounds=([y2[-1]/2, 0.1, 0],[y2[-1]*5 + 1, 0.8, 150]))
y2_pred = test(x2,param2[0], param2[1], param2[2])
 group = test_groups.get_group(index)
 group['days'] = [(datetime.strptime(date, "%Y-%m-%d" ).date() - base_date_object ).days + 1 for date in group['Date'].values]
group = group[group['days'] <= public_end_day]
ids = group['ForecastId'].values
days = [(datetime.strptime(date, "%Y-%m-%d" ).date() - base_date_object ).days + 1 for date in group['Date'].values]
prev_days = range(public_start_day - 6, public_start_day - 1, 1)
 if stage3:
test_y1_pred_raw = custom(days, param[0], param[1], param[2], param[3], param[4], param[5], param[6])
 prev_y1_pred_raw = custom(prev_days, param[0], param[1], param[2], param[3], param[4], param[5], param[6])
 elif index[0] == 'Korea, South':
test_y1_pred_raw = test_linear(days, param[0], param[1], param[2], param[3], param[4], param[5])
 prev_y1_pred_raw = test_linear(prev_days, param[0], param[1], param[2], param[3], param[4], param[5])
 elif index[0] in ['US', 'Spain', 'Germany', 'France', 'Iran', 'United Kingdom']:
test_y1_pred_raw = test(days, param[0], param[1], param[2])
 prev_y1_pred_raw = test(prev_days, param[0], param[1], param[2])
 else:
test_y1_pred_raw = test(days, param[0], param[1], param[2])
 prev_y1_pred_raw = test(prev_days, param[0], param[1], param[2])
 test_y2_pred_raw = test(days, param2[0], param2[1], param2[2])
 prev_y2_pred_raw = test(prev_days, param2[0], param2[1], param2[2])
 day_index_pred = 0
diff1_list = []
diff2_list = []
for day in prev_days:
if day in x:
day_index = np.where(x == day)
 diff1 = y1[day_index] - prev_y1_pred_raw[day_index_pred]
diff1_list.append(diff1)
 if day in x2:
day_index = np.where(x2 == day)
 diff2 = y2[day_index] - prev_y2_pred_raw[day_index_pred]
diff2_list.append(diff2)
 day_index_pred = day_index_pred + 1
if len(diff1_list)> 0:
diff1_mean = np.max(diff1_list)
 else:
diff1_mean = 0
if len(diff2_list)> 0:
diff2_mean = np.max(diff2_list)
 else:
diff2_mean = 0
if np.isnan(diff1_mean):
pass
else:
test_y1_pred_raw = list(np.array(test_y1_pred_raw)+ diff1_mean)
 if np.isnan(diff2_mean):
pass
else:
test_y2_pred_raw = list(np.array(test_y2_pred_raw)+ diff2_mean)
 test_y1_pred = test_y1_pred_raw
test_y2_pred = test_y2_pred_raw
ratio = 0
if y2[-1] > 0:
ratio = y1[-1]/y2[-1]
else:
ratio = y1[-1]
train_day_index = days.index(public_start_day)- 1
if(y1[-1] < 100 and y2[-1] < 4 and ratio < avg):
for pred_index in range(len(test_y2_pred)) :
if pred_index > train_day_index:
if test_y2_pred[pred_index] < test_y1_pred[pred_index]/avg:
test_y2_pred[pred_index] = test_y1_pred[pred_index]/avg
else:
for pred_index in range(len(test_y2_pred)) :
if pred_index > train_day_index:
if test_y2_pred[pred_index] < test_y1_pred[pred_index]/ratio:
test_y2_pred[pred_index] = test_y1_pred[pred_index]/ratio
test_y1_pred = [int(n)for n in test_y1_pred]
test_y2_pred = [int(m)for m in test_y2_pred]
local_df_hr = pd.DataFrame(ids, columns=['ForecastId'])
 print()
local_df_hr.insert(1, 'Country_Region', [index[0]]*len(days))
local_df_hr.insert(2, 'Province_State', [index[1]]*len(days))
local_df_hr.insert(3, 'Days', days)
 local_df_hr.insert(4, 'ConfirmedCases', test_y1_pred)
 local_df_hr.insert(5, 'Fatalities', test_y2_pred)
 local_df_hr.insert(6, 'Date', group['Date'].values)
 local_df = pd.DataFrame(ids, columns=['ForecastId'])
 local_df.insert(1, 'ConfirmedCases', test_y1_pred)
 local_df.insert(2, 'Fatalities', test_y2_pred)
 df = df.append(local_df)
 df_hr = df_hr.append(local_df_hr)
 count = count + 1
train_df['project_descp'] = train_df['project_subject_categories'] + ' ' + train_df['project_subject_subcategories'] + ' ' + train_df['project_title'] + ' ' + train_df['project_resource_summary'] + ' ' + train_df['project_essay']
test_df['project_descp'] = test_df['project_subject_categories'] + ' ' + test_df['project_subject_subcategories'] + ' ' + test_df['project_title'] + ' ' + test_df['project_resource_summary'] + ' ' + test_df['project_essay']
train_df = train_df.drop([ 'project_essay'], axis=1)
 test_df = test_df.drop(['project_essay'], axis=1)
 gc.collect()<categorify>
private_start_date = '2020-04-02'
private_end_date = '2020-04-30'
count = 0
for index, start_date in zip(min_date_sorted.index, min_date_sorted):
print(list(min_date_sorted.index ).index(index), index, start_date)
 record = df_train[(df_train['Country_Region'] == index[0])&(df_train['Province_State'] == index[1])]
if len(record[record['ConfirmedCases'] > 0])== 0:
pass
else:
record = record[record['ConfirmedCases'] > 0]
base_date_object = datetime.strptime(start_date, "%Y-%m-%d" ).date()
private_start_date_object = datetime.strptime(private_start_date, "%Y-%m-%d" ).date()
private_end_date_object = datetime.strptime(private_end_date, "%Y-%m-%d" ).date()
record['days'] = [(datetime.strptime(date, "%Y-%m-%d" ).date() - base_date_object ).days + 1 for date in record['Date']]
private_start_day =(private_start_date_object - base_date_object ).days + 1
private_end_day =(private_end_date_object - base_date_object ).days + 1
if len(record[record['days'] < private_start_day])> 0:
record = record[record['days'] < private_start_day]
record2 = record
if len(record[record['Fatalities'] > 0])!= 0:
record2 = record[record['Fatalities'] > 0]
x = record['days'].values
x2 = record2['days'].values
y1 = record['ConfirmedCases'].values
y2 = record2['Fatalities'].values
b = -1
bad_index = 0
mod_count = 0
for a in y1:
if a < b:
y1[bad_index] = b
mod_count = mod_count + 1
else:
b = a
bad_index = bad_index + 1
if mod_count > 0:
print("*****************")
 print(list(min_date_sorted.index ).index(index), index)
 print(mod_count)
 print(y1)
 print("*****************")
 b = -1
bad_index = 0
mod_count = 0
for a in y2:
if a < b:
y2[bad_index] = b
mod_count = mod_count + 1
else:
b = a
bad_index = bad_index + 1
if mod_count > 0:
print("*****************")
 print(list(min_date_sorted.index ).index(index), index)
 print(mod_count)
 print(y2)
 print("*****************")
 y_max_ = y1[-1]
y1_prime = np.diff(y1)
 y1_prime2 = np.diff(y1_prime)
 if len(y1)>0 and len(y1_prime)> 0 and len(y1_prime2)> 0:
max_slope_index = len(y1_prime)- 1 - list(y1_prime)[::-1].index(max(y1_prime))
max_slope_range =(max_slope_index+1)/len(y1_prime)
 y_max_ = y1[-1]
if max_slope_range < 0.75:
if y1_prime[max_slope_index] > 0 and max_slope_range < 0.75 and(((y1_prime[max_slope_index] - max(y1_prime[-2:])) /y1_prime[max_slope_index])< 0.5):
y_max_ = y1[-1]
pass
else:
y_max_ = y1[max_slope_index + 1]
pass
else:
y_max_ = y1[-1]
stage0 = False
stage1 = False
stage2 = False
stage3 = False
count1 = 0
count2 = 0
for start in range(len(y1_prime)-3):
if sum(y1_prime[start:start+3])<=12:
count1 = count1 + 1
count2 = 0
else:
count2 = count2 + 1
count1 = 0
if not stage0 and count2 == 0 and count1 > 2:
stage0 = True
count1 = 0
if not stage1 and count1 == 0 and count2 > 5:
stage0 = True
stage1 = True
count2 = 0
if stage1 and count2 == 0 and count1 > 3:
stage2 = True
count1 = 0
if stage2 and count1 == 0 and count2 > 2:
stage3 = True
count2 = 0
if stage3:
param, param_cov = curve_fit(custom, np.array(x), np.array(y1), maxfev = 100000, bounds=([1, 0, 1, 1, 30, 1, 1], [2, 60, 8, 200, 90, 8, 1400]))
y1_pred = custom(x, param[0], param[1], param[2], param[3], param[4], param[5], param[6])
 elif index[0] == 'Korea, South':
param, param_cov = curve_fit(test_linear, np.array(x), np.array(y1), [y_max_, 0.5,(x[-1] - x[0])/2 + x[0], 50, 45, 0], maxfev = 100000, bounds=([y_max_/2, 0.1, 0, 1, 30, -100],[y_max_*5 + 1500, 1, 150, 100, 100, 1000]))
y1_pred = test_linear(x, param[0], param[1], param[2], param[3], param[4], param[5])
 elif index[0] in ['US', 'Spain', 'Germany', 'France', 'Iran', 'United Kingdom']:
param, param_cov = curve_fit(test, np.array(x), np.array(y1), [y_max_*6, 0.5,(x[-1] - x[0])/2 + x[0]], maxfev = 100000, bounds=([y_max_*5, 0.1, 0],[y_max_*10 + 1500, 1, 150]))
y1_pred = test(x, param[0], param[1], param[2])
 elif index[0] == 'China':
param, param_cov = curve_fit(test, np.array(x), np.array(y1), [y_max_, 0.5,(x[-1] - x[0])/2 + x[0]], maxfev = 100000, bounds=([y_max_/2, 0.1, 0],[y_max_*5 + 1500, 1, 150]))
y1_pred = test(x, param[0], param[1], param[2])
 elif index[0] in ['Italy', 'Switzerland']:
param, param_cov = curve_fit(test, np.array(x), np.array(y1), [y_max_*3, 0.5,(x[-1] - x[0])/2 + x[0]], maxfev = 100000, bounds=([y_max_*2, 0.1, 0],[y_max_*5 + 1500, 1, 150]))
y1_pred = test(x, param[0], param[1], param[2])
 else:
param, param_cov = curve_fit(test, np.array(x), np.array(y1), [y_max_*4, 0.5,(x[-1] - x[0])/2 + x[0]], maxfev = 100000, bounds=([y_max_*3, 0.1, 0],[y_max_*8 + 1500, 1, 150]))
y1_pred = test(x, param[0], param[1], param[2])
 param2, param_cov2 = curve_fit(test, np.array(x2), np.array(y2), [y2[-1]/2, 0.5,(x2[-1] - x2[0])/2 + x2[0] - 3], maxfev = 100000, bounds=([y2[-1]/2, 0.1, 0],[y2[-1]*5 + 1, 0.8, 150]))
y2_pred = test(x2,param2[0], param2[1], param2[2])
 group = test_groups.get_group(index)
 group['days'] = [(datetime.strptime(date, "%Y-%m-%d" ).date() - base_date_object ).days + 1 for date in group['Date'].values]
group = group[group['days'] >= private_start_day]
ids = group['ForecastId'].values
days = [(datetime.strptime(date, "%Y-%m-%d" ).date() - base_date_object ).days + 1 for date in group['Date'].values]
prev_days = range(private_start_day - 6, private_start_day - 1, 1)
 if stage3:
test_y1_pred_raw = custom(days, param[0], param[1], param[2], param[3], param[4], param[5], param[6])
 prev_y1_pred_raw = custom(prev_days, param[0], param[1], param[2], param[3], param[4], param[5], param[6])
 elif index[0] == 'Korea, South':
test_y1_pred_raw = test_linear(days, param[0], param[1], param[2], param[3], param[4], param[5])
 prev_y1_pred_raw = test_linear(prev_days, param[0], param[1], param[2], param[3], param[4], param[5])
 elif index[0] in ['US', 'Spain', 'Germany', 'France', 'Iran', 'United Kingdom']:
test_y1_pred_raw = test(days, param[0], param[1], param[2])
 prev_y1_pred_raw = test(prev_days, param[0], param[1], param[2])
 else:
test_y1_pred_raw = test(days, param[0], param[1], param[2])
 prev_y1_pred_raw = test(prev_days, param[0], param[1], param[2])
 test_y2_pred_raw = test(days, param2[0], param2[1], param2[2])
 prev_y2_pred_raw = test(prev_days, param2[0], param2[1], param2[2])
 day_index_pred = 0
diff1_list = []
diff2_list = []
for day in prev_days:
if day in x:
day_index = np.where(x == day)
 diff1 = y1[day_index] - prev_y1_pred_raw[day_index_pred]
diff1_list.append(diff1)
 if day in x2:
day_index = np.where(x2 == day)
 diff2 = y2[day_index] - prev_y2_pred_raw[day_index_pred]
diff2_list.append(diff2)
 day_index_pred = day_index_pred + 1
if len(diff1_list)> 0:
diff1_mean = np.max(diff1_list)
 else:
diff1_mean = 0
if len(diff2_list)> 0:
diff2_mean = np.max(diff2_list)
 else:
diff2_mean = 0
if np.isnan(diff1_mean):
pass
else:
test_y1_pred_raw = list(np.array(test_y1_pred_raw)+ diff1_mean)
 if np.isnan(diff2_mean):
pass
else:
test_y2_pred_raw = list(np.array(test_y2_pred_raw)+ diff2_mean)
 test_y1_pred = test_y1_pred_raw
test_y2_pred = test_y2_pred_raw
ratio = 0
if y2[-1] > 0:
ratio = y1[-1]/y2[-1]
else:
ratio = y1[-1]
train_day_index = days.index(private_start_day)- 1
if(y1[-1] < 100 and y2[-1] < 4 and ratio < avg):
for pred_index in range(len(test_y2_pred)) :
if pred_index > train_day_index:
if test_y2_pred[pred_index] < test_y1_pred[pred_index]/avg:
test_y2_pred[pred_index] = test_y1_pred[pred_index]/avg
else:
for pred_index in range(len(test_y2_pred)) :
if pred_index > train_day_index:
if test_y2_pred[pred_index] < test_y1_pred[pred_index]/ratio:
test_y2_pred[pred_index] = test_y1_pred[pred_index]/ratio
test_y1_pred = [int(n)for n in test_y1_pred]
test_y2_pred = [int(m)for m in test_y2_pred]
local_df_hr = pd.DataFrame(ids, columns=['ForecastId'])
 local_df_hr.insert(1, 'Country_Region', [index[0]]*len(days))
local_df_hr.insert(2, 'Province_State', [index[1]]*len(days))
local_df_hr.insert(3, 'Days', days)
 local_df_hr.insert(4, 'ConfirmedCases', test_y1_pred)
 local_df_hr.insert(5, 'Fatalities', test_y2_pred)
 local_df_hr.insert(6, 'Date', group['Date'].values)
 local_df = pd.DataFrame(ids, columns=['ForecastId'])
 local_df.insert(1, 'ConfirmedCases', test_y1_pred)
 local_df.insert(2, 'Fatalities', test_y2_pred)
 df = df.append(local_df)
 df_hr = df_hr.append(local_df_hr)
 count = count + 1
label_cols = [
'teacher_prefix',
'school_state',
'project_grade_category',
'project_subject_categories',
'project_subject_subcategories'
] + time_cols
def mean_encoding(train_df, test_df, col):
gp = train_df.groupby(col)['project_is_approved'].mean().reset_index().rename(columns={'project_is_approved': col+'_mean'})
 train_df = pd.merge(train_df, gp, how='left', on=[col])
 test_df = pd.merge(test_df, gp, how='left', on=[col])
 return train_df, test_df
for col in label_cols:
train_df, test_df = mean_encoding(train_df, test_df, col)
 label_mean_cols = [item+'_mean' for item in label_cols]
train_label_features = train_df[label_mean_cols].values
test_df[label_mean_cols] = test_df[label_mean_cols].fillna(test_df[label_mean_cols].mean())
 test_label_features = test_df[label_mean_cols].values

 train_df = train_df.drop(label_cols, axis=1)
 test_df = test_df.drop(label_cols, axis=1)
 train_df = train_df.drop(label_mean_cols, axis=1)
 test_df = test_df.drop(label_mean_cols, axis=1)
 gc.collect()<feature_engineering>
df = df.sort_values(by=['ForecastId'], ascending=True)
 df_hr = df_hr.sort_values(by=['ForecastId'], ascending=True )
<feature_engineering><EOS> 
df.to_csv('submission.csv', index=False)
 df_hr.to_csv('hr_submission.csv', index=False )