Kaggle competition: COVID19 Global Forecasting (Week 1)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-1)<load_from_csv>
import pandas as pd
import datetime
import lightgbm as lgb
BATCH_SIZE = 512
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
EPOCHS = 4
MAX_LEN = 220
TEXT_COLUMN = 'comment_text'
list_classes = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]
CHARS_TO_REMOVE = '!"
“”’'∞θ÷α•à−β∅³π‘₹´°£€\×™√²—'
submission = pd.read_csv(".. /input/jigsaw-toxic-comment-classification-challenge/sample_submission.csv" )<define_variables>
train = pd.read_csv(".. /input/covid19-global-forecasting-week-1/train.csv")
 test = pd.read_csv(".. /input/covid19-global-forecasting-week-1/test.csv")
 sub = pd.read_csv(".. /input/covid19-global-forecasting-week-1/submission.csv" )
categories = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]
data_folder = ".. /input/jigsaw-toxic-comment-classification-challenge/"
pretrained_folder = ".. /input/"
train_filepath = data_folder + "train.csv"
test_filepath = data_folder + "test.csv"
submission_path = data_folder + "submission.csv"
facebook_filepath = pretrained_folder + "fasttext-crawl-300d-2m/crawl-300d-2M.vec"
google_filepath = pretrained_folder + "glove840b300dtxt/glove.840B.300d.txt"
embedding_dim_facebook = 300
embedding_dim_google = 300<load_pretrained>
train.loc[(train['Date']=='2020-03-24')&(train['Country/Region']=='France')&(train['Province/State']=='France'),'ConfirmedCases'] = 22654
train.loc[(train['Date']=='2020-03-24')&(train['Country/Region']=='France')&(train['Province/State']=='France'),'Fatalities'] = 1000
hyphens_filepath = ".. /input/cleaning-dictionaries/hyphens_dictionary.bin"
misspellings_filepath = ".. /input/cleaning-dictionaries/misspellings_all_dictionary.bin"
merged_filepath = ".. /input/cleaning-dictionaries/merged_all_dictionary.bin"
hyphens_dict = misspellings_dict = merged_dict = {}
with open(hyphens_filepath, mode='rb')as file: hyphens_dict = pickle.load(file)
 with open(misspellings_filepath, mode='rb')as file: misspellings_dict = pickle.load(file)
 with open(merged_filepath, mode='rb')as file: merged_dict = pickle.load(file)
 print(len(hyphens_dict))
print(len(misspellings_dict))
print(len(merged_dict))<define_variables>
train[(train['Date']=='2020-03-24')&(train['Country/Region']=='France')&(train['Province/State']=='France')]
training_samples_count = 149571
validation_samples_count = 10000
length_threshold = 20000
word_count_threshold = 900
words_limit = 310000
valid_characters = " " + "@$" + "'!?-" + "abcdefghijklmnopqrstuvwxyz" + "abcdefghijklmnopqrstuvwxyz".upper()
valid_characters_ext = valid_characters + "abcdefghijklmnopqrstuvwxyz".upper()
valid_set = set(x for x in valid_characters)
 valid_set_ext = set(x for x in valid_characters_ext)
 toxic_words = ["poop", "crap", "prick", "twat", "wikipedia", "wiki", "hahahahaha", "lol", "bastard", "sluts", "slut", "douchebag", "douche", "blowjob", "nigga", "dumb", "jerk", "wanker", "wank", "penis", "motherfucker", "fucker", "fuk", "fucking", "fucked", "fuck", "bullshit", "shit", "stupid", "bitches", "bitch", "suck", "cunt", "dick", "cocks", "cock", "die", "kill", "gay", "jewish", "jews", "jew", "niggers", "nigger", "faggot", "fag", "asshole"]
astericks_words = [('mother****ers', 'motherfuckers'),('motherf*cking', 'motherfucking'),('mother****er', 'motherfucker'),('motherf*cker', 'motherfucker'),('bullsh*t', 'bullshit'),('f**cking', 'fucking'),('f*ucking', 'fucking'),('fu*cking', 'fucking'),('****ing', 'fucking'),('a**hole', 'asshole'),('assh*le', 'asshole'),('f******', 'fucking'),('f*****g', 'fucking'),('f***ing', 'fucking'),('f**king', 'fucking'),('f*cking', 'fucking'),('fu**ing', 'fucking'),('fu*king', 'fucking'),('fuc*ers', 'fuckers'),('f*****', 'fucking'),('f***ed', 'fucked'),('f**ker', 'fucker'),('f*cked', 'fucked'),('f*cker', 'fucker'),('f*ckin', 'fucking'),('fu*ker', 'fucker'),('fuc**n', 'fucking'),('ni**as', 'niggas'),('b**ch', 'bitch'),('b*tch', 'bitch'),('c*unt', 'cunt'),('f**ks', 'fucks'),('f*ing', 'fucking'),('ni**a', 'nigga'),('c*ck', 'cock'),('c*nt', 'cunt'),('cr*p', 'crap'),('d*ck', 'dick'),('f***', 'fuck'),('f**k', 'fuck'),('f*ck', 'fuck'),('fc*k', 'fuck'),('fu**', 'fuck'),('fu*k', 'fuck'),('s***', 'shit'),('s**t', 'shit'),('sh**', 'shit'),('sh*t', 'shit'),('tw*t', 'twat')]
fasttext_misspelings = {"'n'balls": 'balls', "-nazi's": 'nazis', 'adminabuse': 'admin abuse', "admins's": 'admins', 'arsewipe': 'arse wipe', 'assfack': 'asshole', 'assholifity': 'asshole', 'assholivity': 'asshole', 'asshoul': 'asshole', 'asssholeee': 'asshole', 'belizeans': 'mexicans', "blowing's": 'blowing', 'bolivians': 'mexicans', 'celtofascists': 'fascists', 'censorshipmeisters': 'censor', 'chileans': 'mexicans', 'clerofascist': 'fascist', 'cowcrap': 'crap', 'crapity': 'crap', "d'idiots": 'idiots', 'deminazi': 'nazi', 'dftt': "don't feed the troll", 'dildohs': 'dildo', 'dramawhores': 'drama whores', 'edophiles': 'pedophiles', 'eurocommunist': 'communist', 'faggotkike': 'faggot', 'fantard': 'retard', 'fascismnazism': 'fascism', 'fascistisized': 'fascist', 'favremother': 'mother', 'fuxxxin': 'fucking', "g'damn": 'goddamn', 'harassmentat': 'harassment', 'harrasingme': 'harassing me', 'herfuc': 'motherfucker', 'hilterism': 'fascism', 'hitlerians': 'nazis', 'hitlerites': 'nazis', 'hubrises': 'pricks', 'idiotizing': 'idiotic', 'inadvandals': 'vandals', "jackass's": 'jackass', 'jiggabo': 'nigga', 'jizzballs': 'jizz balls', 'jmbass': 'dumbass', 'lejittament': 'legitimate', "m'igger": 'nigger', "m'iggers": 'niggers', 'motherfacking': 'motherfucker', 'motherfuckenkiwi': 'motherfucker', 'muthafuggas': 'niggas', 'nazisms': 'nazis', 'netsnipenigger': 'nigger', 'niggercock': 'nigger', 'niggerspic': 'nigger', 'nignog': 'nigga', 'niqqass': 'niggas', "non-nazi's": 'not a nazi', 'panamanians': 'mexicans', 'pedidiots': 'idiots', 'picohitlers': 'hitler', 'pidiots': 'idiots', 'poopia': 'poop', 'poopsies': 'poop', 'presumingly': 'obviously', 'propagandaanddisinformation': 'propaganda and disinformation', 'propagandaministerium': 'propaganda', 'puertoricans': 'mexicans', 'puertorricans': 'mexicans', 'pussiest': 'pussies', 'pussyitis': 'pussy', 'rayaridiculous': 'ridiculous', 'redfascists': 'fascists', 'retardzzzuuufff': 'retard', "revertin'im": 'reverting', 'scumstreona': 'scums', 'southamericans': 'mexicans', 'strasserism': 'fascism', 'stuptarded': 'retarded', "t'nonsense": 'nonsense', "threatt's": 'threat', 'titoists': 'communists', 'twatbags': 'douchebags', 'youbollocks': 'you bollocks'}
acronym_words = {}<define_variables>
train = train.append(test[test['Date']>'2020-03-24'] )
cont_patterns = [ 
(r'(W|w)on't', r'will not'), 
(r'(C|c)an't', r'can not'), 
(r'(I|i)'m', r'i am'), 
(r'(A|a)in't', r'is not'), 
(r'(\w+)'ll', r'\g<1> will'), 
(r'(\w+)n't', r'\g<1> not'), 
(r'(\w+)'ve', r'\g<1> have'), 
(r'(\w+)'s', r'\g<1> is'), 
(r'(\w+)'re', r'\g<1> are'), 
(r'(\w+)'d', r'\g<1> would'),
]
patterns = [(re.compile(regex), repl)for(regex, repl)in cont_patterns]
def split_word(word, toxic_words):
if word == "":
return ""
lower = word.lower()
for toxic_word in toxic_words:
start = lower.find(toxic_word)
 if start >= 0:
end = start + len(toxic_word)
 result = " ".join([word[0:start], word[start:end], split_word(word[end:], toxic_words)])
 return result.replace(" ", " " ).strip()
return word
tknzr = TweetTokenizer(strip_handles=False, reduce_len=True)
 def word_tokenize(sentence):
sentence = sentence.replace("$", "s")
 sentence = sentence.replace("@", "a")
 sentence = sentence.replace("!", " ! ")
 sentence = sentence.replace("?", " ? ")
 return tknzr.tokenize(sentence)
 def replace_url(word):
if "http://" in word or "www." in word or "https://" in word or "wikipedia.org" in word:
return ""
return word
def normalize_by_dictionary(normalized_word, dictionary):
result = []
for word in normalized_word.split() :
if word == word.upper() :
if word.lower() in dictionary:
result.append(dictionary[word.lower() ].upper())
 else:
result.append(word)
 else:
if word.lower() in dictionary:
result.append(dictionary[word.lower() ])
 else:
result.append(word)
 return " ".join(result )<categorify>
train['Date'] = pd.to_datetime(train['Date'], format='%Y-%m-%d')
nlp = spacy.load('en')
 def normalize_comment(comment):
comment = unidecode(comment)
 comment = comment[:length_threshold]
normalized_words = []
for w in astericks_words:
if w[0] in comment:
comment = comment.replace(w[0], w[1])
 if w[0].upper() in comment:
comment = comment.replace(w[0].upper() , w[1].upper())
 for word in word_tokenize(comment):
if word == "." or word == ",":
normalized_words.append(word)
 continue
word = replace_url(word)
 if word.count(".")== 1:
word = word.replace(".", " ")
 filtered_word = "".join([x for x in word if x in valid_set])
 normalized_word = split_word(filtered_word, toxic_words)
 normalized_word = normalize_by_dictionary(normalized_word, hyphens_dict)
 normalized_word = normalize_by_dictionary(normalized_word, merged_dict)
 normalized_word = normalize_by_dictionary(normalized_word, misspellings_dict)
 normalized_word = normalize_by_dictionary(normalized_word, fasttext_misspelings)
 normalized_word = normalize_by_dictionary(normalized_word, acronym_words)
 normalized_words.append(normalized_word)
 normalized_comment = " ".join(normalized_words)
 result = []
for word in normalized_comment.split() :
if word.upper() == word:
result.append(word)
 else:
result.append(word.lower())
 result = " ".join(result)
 if "sock puppet" in result:
result = result.replace("sock puppet", "sockpuppet")
 if "SOCK PUPPET" in result:
result = result.replace("SOCK PUPPET", "SOCKPUPPET")
 return result 
<load_from_csv>
train['day_dist'] = train['Date']-train['Date'].min()
def read_data_files(train_filepath, test_filepath):
train = pd.read_csv(train_filepath)
 labels = train[categories].values
test = pd.read_csv(test_filepath)
 test_comments = test["comment_text"].fillna("_na_" ).values
np_normalize = np.vectorize(normalize_comment)
 comments = train["comment_text"].fillna("_na_" ).values
normalized_comments = np_normalize(comments)
 del comments
gc.collect()
comments = test["comment_text"].fillna("_na_" ).values
normalized_test_comments = np_normalize(test_comments)
 del comments
gc.collect()
print('Shape of data tensor:', normalized_comments.shape)
 print('Shape of label tensor:', labels.shape)
 print('Shape of test data tensor:', normalized_test_comments.shape)
 return(labels, normalized_comments, normalized_test_comments )<load_from_csv>
train['day_dist'] = train['day_dist'].dt.days
%%time
labels, x_train, x_test = read_data_files(train_filepath, test_filepath )<string_transform>
cat_cols = train.dtypes[train.dtypes=='object'].keys()
cat_cols
%%time
fileObject = open('.. /input/cleaning-dictionaries/tokenizer','rb')
 tokenizer = pickle.load(fileObject)
 x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_train = sequence.pad_sequences(x_train, maxlen=MAX_LEN)
 x_test = sequence.pad_sequences(x_test, maxlen=MAX_LEN )<load_pretrained>
for cat_col in cat_cols:
train[cat_col].fillna('no_value', inplace = True )
%%time
embedding_matrix = np.load('.. /input/embedding-2/embedding_matrix_big.npy' )<split>
train['place'] = train['Province/State']+'_'+train['Country/Region']
x_train, x_valid, y_train, y_valid = train_test_split(x_train, labels, test_size = 0.1)
<choose_model_class>
from sklearn import preprocessing
def build_model(embedding_matrix):
words = Input(shape=(None,))
x = Embedding(*embedding_matrix.shape, weights=[embedding_matrix], trainable=False )(words)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 hidden = concatenate([
GlobalMaxPooling1D()(x),
GlobalAveragePooling1D()(x),
])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 result = Dense(6, activation='sigmoid' )(hidden)
 model = Model(inputs=words, outputs=result)
 model.compile(loss='binary_crossentropy', optimizer='adam')
 return model<train_model>
cat_cols = train.dtypes[train.dtypes=='object'].keys()
cat_cols
EPOCHS = 5
SEEDS = 10
pred = 0
for ii in range(SEEDS):
model = build_model(embedding_matrix)
 for global_epoch in range(EPOCHS):
print(global_epoch)
 model.fit(
 x_train,
y_train,
validation_data =(x_valid, y_valid),
batch_size=128,
epochs=1,
verbose=2,
callbacks=[
LearningRateScheduler(lambda _: 1e-3 *(0.5 ** global_epoch))
] 
)
 val_preds = model.predict(x_valid)
 AUC = 0
for i in range(6):
AUC += roc_auc_score(y_valid[:,i], val_preds[:,i])/6.
 print(AUC)
 pred += model.predict(x_test, batch_size = 1024, verbose = 1)/SEEDS
np.save('pred', pred)
 model.save_weights('model_weights_'+str(ii)+'.h5')
 os.system('gzip '+'model_weights_'+str(ii)+'.h5' )<save_to_csv>
for cat_col in ['place']:
le = preprocessing.LabelEncoder()
le.fit(train[cat_col])
 train[cat_col]=le.transform(train[cat_col] )
submission[list_classes] =(pred)
 submission.to_csv("submission.csv", index = False )<set_options>
drop_cols = ['Id','ForecastId', 'ConfirmedCases','Date', 'Fatalities','day_dist', 'Province/State', 'Country/Region']
print(tf.__version__)
 tf.test.is_gpu_available(
 cuda_only=False,
min_cuda_compute_capability=None
)<define_variables>
val = train[(train['Date']>='2020-03-12')&(train['Id'].isnull() ==False)]
EMBEDDING_FILES = [
'.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec',
'.. /input/glove840b300dtxt/glove.840B.300d.txt'
]
BATCH_SIZE = 512
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
EPOCHS = 4
MAX_LEN = 220
TEXT_COLUMN = 'comment_text'
list_classes = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]
CHARS_TO_REMOVE = '!"
“”’'∞θ÷α•à−β∅³π‘₹´°£€\×™√²—'<load_from_csv>
y_ft = train["Fatalities"]
y_val_ft = val["Fatalities"]
y_cc = train["ConfirmedCases"]
y_val_cc = val["ConfirmedCases"]
train_df = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/train.csv')
 test_df = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/test.csv')
 submission = pd.read_csv(".. /input/jigsaw-toxic-comment-classification-challenge/sample_submission.csv")
 y = train_df[list_classes].values
x_train = train_df[TEXT_COLUMN].astype(str)
 y_train = y
x_test = test_df[TEXT_COLUMN].astype(str )<categorify>
def rmsle(y_true, y_pred):
return np.sqrt(np.mean(np.power(np.log1p(y_pred)- np.log1p(y_true), 2)))
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 def load_embeddings(path):
with open(path)as f:
return dict(get_coefs(*line.strip().split(' ')) for line in f)
 def build_matrix(word_index, path):
embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( len(word_index)+ 1, 300))
for word, i in word_index.items() :
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
pass
return embedding_matrix
def build_model(embedding_matrix):
words = Input(shape=(None,))
x = Embedding(*embedding_matrix.shape, weights=[embedding_matrix], trainable=False )(words)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 hidden = concatenate([
GlobalMaxPooling1D()(x),
GlobalAveragePooling1D()(x),
])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 result = Dense(6, activation='sigmoid' )(hidden)
 model = Model(inputs=words, outputs=result)
 model.compile(loss='binary_crossentropy', optimizer='adam')
 return model<string_transform>
def mape(y_true, y_pred):
return np.mean(np.abs(y_pred -y_true)*100/(y_true+1))
%%time
tokenizer = text.Tokenizer(filters=CHARS_TO_REMOVE)
 tokenizer.fit_on_texts(list(x_train)+ list(x_test))
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_train = sequence.pad_sequences(x_train, maxlen=MAX_LEN)
 x_test = sequence.pad_sequences(x_test, maxlen=MAX_LEN )<concatenate>
import numpy as np
%%time
embedding_matrix = np.concatenate(
 [build_matrix(tokenizer.word_index, f)for f in EMBEDDING_FILES], axis=-1 )<split>
params = {
"objective": "regression",
"boosting": 'gbdt',
"num_leaves": 1280,
"learning_rate": 0.05,
"feature_fraction": 0.9,
"reg_lambda": 2,
"metric": "rmse",
'min_data_in_leaf':20
}
X_train, X_valid, Y_train, Y_valid = train_test_split(x_train, y_train, test_size = 0.1)
<train_model>
dates = dates[dates>'2020-03-24']
EPOCHS = 5
SEEDS = 10
pred = 0
for ii in range(SEEDS):
model = build_model(embedding_matrix)
 for global_epoch in range(EPOCHS):
print(global_epoch)
 model.fit(
 X_train,
Y_train,
validation_data =(X_valid, Y_valid),
batch_size=128,
epochs=1,
verbose=2,
callbacks=[
LearningRateScheduler(lambda _: 1e-3 *(0.55 ** global_epoch))
] 
)
 val_preds_3 = model.predict(X_valid)
 AUC = 0
for i in range(6):
AUC += roc_auc_score(Y_valid[:,i], val_preds_3[:,i])/6.
 print(AUC)
 pred += model.predict(x_test, batch_size = 1024, verbose = 1)/SEEDS
model.save_weights('model_weights_'+str(ii)+'.h5')
 os.system('gzip '+'model_weights_'+str(ii)+'.h5')
<save_to_csv>
test[test['Country/Region']=='Italy']
submission[list_classes] =(pred)
 submission.to_csv("submission.csv", index = False )<import_modules>
test[test['Country/Region']=='Italy']
import tensorflow as tf
import matplotlib.pyplot as plt
from tensorflow import keras
from keras.layers import Flatten
from keras.layers import Conv2D
from keras.layers import MaxPool2D
from keras.layers import Dense
from keras.layers import Dropout
from keras.layers import BatchNormalization
from keras.layers import Average
from keras.layers import Activation
from tensorflow.keras import layers<load_from_csv>
train_sub = pd.read_csv(".. /input/covid19-global-forecasting-week-1/train.csv" )
def load_data(do_reshape=False):
DMNIST_pd = pd.read_csv("/kaggle/input/Kannada-MNIST/Dig-MNIST.csv")
 train_pd = pd.read_csv("/kaggle/input/Kannada-MNIST/train.csv")
 test_pd = pd.read_csv("/kaggle/input/Kannada-MNIST/test.csv")
 x_train_np, y_train_np = train_pd.iloc[:,1:].to_numpy() , train_pd.iloc[:,0].to_numpy()
x_DMNIST_np, y_DMNIST_np = DMNIST_pd.iloc[:,1:].to_numpy() , DMNIST_pd.iloc[:,0].to_numpy()
x_data=np.concatenate(( x_train_np,x_DMNIST_np))
y_data=np.concatenate(( y_train_np,y_DMNIST_np))
x_test_full = test_pd.iloc[:,1:].to_numpy()
x_train_full, x_valid_full, y_train_full, y_valid_full = train_test_split(x_data, y_data,test_size=0.2, random_state=42)
 if do_reshape:
x_train_full = np.reshape(x_train_full,(-1, 28, 28, 1))
x_valid_full = np.reshape(x_valid_full,(-1, 28, 28, 1))
x_test_full = np.reshape(x_test_full,(-1, 28, 28, 1))
return(x_train_full, y_train_full),(x_valid_full, y_valid_full), x_test_full
def show_imgs(x_train, y_train, rows=4):
classes = ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"]
print(classes)
 for y, cls in enumerate(classes):
idxs = np.nonzero([i == y for i in y_train])
 idxs = np.random.choice(idxs[0], rows)
 for i , idx in enumerate(idxs):
plt_idx = i * len(classes)+ y + 1
plt.subplot(rows, len(classes), plt_idx)
 plt.imshow(x_train[idx].reshape(( 28, 28)))
 plt.axis("off")
 if i == 0:
plt.title(cls)
 plt.show()<train_model>
train_sub.loc[(train_sub['Date']=='2020-03-24')&(train_sub['Country/Region']=='France')&(train_sub['Province/State']=='France'),'ConfirmedCases'] = 22654
train_sub.loc[(train_sub['Date']=='2020-03-24')&(train_sub['Country/Region']=='France')&(train_sub['Province/State']=='France'),'Fatalities'] = 1000
( x_train, y_train),(x_valid, y_valid), x_test = load_data(do_reshape=True)
 show_imgs(x_train, y_train, rows=4)
 print("Train shape", x_train.shape, y_train.shape)
 print("Valid shape", x_valid.shape, y_valid.shape)
 print("Test shape", x_test.shape )<choose_model_class>
test = pd.merge(test,train_sub[['Province/State','Country/Region','Lat','Long','Date','ConfirmedCases','Fatalities']], on=['Province/State','Country/Region','Lat','Long','Date'], how='left' )
class average_CNN(tf.keras.Model):
def __init__(self, feature_maps = 32, kernel_sizes =(1,3,5), **kwargs):
super(average_CNN, self ).__init__(**kwargs)
 self.cnn_layers = []
for kernel_size in kernel_sizes:
self.cnn_layers.append(
 [Conv2D(feature_maps, kernel_size, activation="relu", kernel_initializer="he_uniform", padding="same"),
Conv2D(feature_maps, kernel_size, activation="relu", kernel_initializer="he_uniform", padding="same"),
BatchNormalization()
] 
)
 self.avg_layer = Average()
self.act = Activation('relu')
 self.bn4 = BatchNormalization()
self.pool = MaxPool2D()
self.drop = Dropout(0.2)
 def call(self, inputs):
x = inputs
self.middle_outs = []
for layer_set in self.cnn_layers:
for layer in layer_set:
x = layer(x)
 self.middle_outs.append(x)
 x = self.avg_layer(self.middle_outs)
 x = self.act(x)
 x = self.bn4(x)
 x = self.pool(x)
 x = self.drop(x)
 return x
class my_CNN(tf.keras.Model):
def __init__(self, **kwargs):
super(my_CNN, self ).__init__(**kwargs)
 self.avg_c1 = average_CNN(32,(1,3,5))
self.avg_c2 = average_CNN(64,(1,3,5))
self.avg_c3 = average_CNN(128,(1,3,5))
self.flat = Flatten()
self.den1 = Dense(128, activation="relu", kernel_initializer="he_uniform")
 self.drop = Dropout(0.2)
 self.den2 = Dense(10, activation="softmax")
 def call(self, inputs):
x = self.avg_c1(inputs)
 x = self.avg_c2(x)
 x = self.avg_c3(x)
 x = self.flat(x)
 x = self.den1(x)
 x = self.drop(x)
 x = self.den2(x)
 return x 
<choose_model_class>
test.loc[test['ConfirmedCases_x'].isnull() ==True]
def module_CNN() :
my_cnn = my_CNN()
my_cnn.compile(loss="sparse_categorical_crossentropy",
optimizer="adam",
metrics=["accuracy"])
 return my_cnn<choose_model_class>
test.loc[test['ConfirmedCases_x'].isnull() ==True, 'ConfirmedCases_x'] = test.loc[test['ConfirmedCases_x'].isnull() ==True, 'ConfirmedCases_y']
def model_CNN() :
model_input = keras.Input(shape=(28,28,1))
C1_1 = Conv2D(32, 1, activation="relu", kernel_initializer="he_uniform", padding="same" )(model_input)
 C1_2 = Conv2D(32, 1, activation="relu", kernel_initializer="he_uniform", padding="same" )(C1_1)
 C1_3 = BatchNormalization()(C1_2)
 C2_1 = Conv2D(32, 3, activation="relu", kernel_initializer="he_uniform", padding="same" )(model_input)
 C2_2 = Conv2D(32, 3, activation="relu", kernel_initializer="he_uniform", padding="same" )(C2_1)
 C2_3 = BatchNormalization()(C2_2)
 C3_1 = Conv2D(32, 5, activation="relu", kernel_initializer="he_uniform", padding="same" )(model_input)
 C3_2 = Conv2D(32, 5, activation="relu", kernel_initializer="he_uniform", padding="same" )(C3_1)
 C3_3 = BatchNormalization()(C3_2)
 avg1_1 = Average()([C1_3, C2_3, C3_3])
 avg1_2 = Activation('relu' )(avg1_1)
 avg1_3 = BatchNormalization()(avg1_2)
 mp1 = MaxPool2D()(avg1_3)
 out1 = Dropout(0.2 )(mp1)
 C1_1 = Conv2D(64, 1, activation="relu", kernel_initializer="he_uniform", padding="same" )(out1)
 C1_2 = Conv2D(64, 1, activation="relu", kernel_initializer="he_uniform", padding="same" )(C1_1)
 C1_3 = BatchNormalization()(C1_2)
 C2_1 = Conv2D(64, 3, activation="relu", kernel_initializer="he_uniform", padding="same" )(out1)
 C2_2 = Conv2D(64, 3, activation="relu", kernel_initializer="he_uniform", padding="same" )(C2_1)
 C2_3 = BatchNormalization()(C2_2)
 C3_1 = Conv2D(64, 5, activation="relu", kernel_initializer="he_uniform", padding="same" )(out1)
 C3_2 = Conv2D(64, 5, activation="relu", kernel_initializer="he_uniform", padding="same" )(C3_1)
 C3_3 = BatchNormalization()(C3_2)
 avg2_1 = Average()([C1_3, C2_3, C3_3])
 avg2_2 = Activation('relu' )(avg2_1)
 avg2_3 = BatchNormalization()(avg2_2)
 mp2 = MaxPool2D()(avg2_3)
 out2 = Dropout(0.2 )(mp2)
 C1_1 = Conv2D(128, 1, activation="relu", kernel_initializer="he_uniform", padding="same" )(out2)
 C1_2 = Conv2D(128, 1, activation="relu", kernel_initializer="he_uniform", padding="same" )(C1_1)
 C1_3 = BatchNormalization()(C1_2)
 C2_1 = Conv2D(128, 3, activation="relu", kernel_initializer="he_uniform", padding="same" )(out2)
 C2_2 = Conv2D(128, 3, activation="relu", kernel_initializer="he_uniform", padding="same" )(C2_1)
 C2_3 = BatchNormalization()(C2_2)
 C3_1 = Conv2D(128, 5, activation="relu", kernel_initializer="he_uniform", padding="same" )(out2)
 C3_2 = Conv2D(128, 5, activation="relu", kernel_initializer="he_uniform", padding="same" )(C3_1)
 C3_3 = BatchNormalization()(C3_2)
 avg3_1 = Average()([C1_3, C2_3, C3_3])
 avg3_2 = Activation('relu' )(avg3_1)
 avg3_3 = BatchNormalization()(avg3_2)
 mp3 = MaxPool2D()(avg3_3)
 out3 = Dropout(0.2 )(mp3)
 out = Flatten()(out3)
 out = Dense(128, activation="relu", kernel_initializer="he_uniform" )(out)
 out = Dropout(0.2 )(out)
 out = Dense(10, activation="softmax" )(out)
 model = tf.keras.Model(inputs=[model_input],outputs=[out])
 model.compile(loss="sparse_categorical_crossentropy",
optimizer="adam",
metrics=["accuracy"])
 return model 
<set_options>
test.loc[test['Fatalities_x'].isnull() ==True, 'Fatalities_x'] = test.loc[test['Fatalities_x'].isnull() ==True, 'Fatalities_y']
rescale_layer = tf.keras.Sequential([layers.experimental.preprocessing.Rescaling(1./255)])
 resize_and_rescale = tf.keras.Sequential([
layers.experimental.preprocessing.Rescaling(1./255)
 ])
 data_augmentation = tf.keras.Sequential([
layers.experimental.preprocessing.RandomContrast(0.5),
layers.experimental.preprocessing.RandomRotation(factor =(0.2), seed=42),
layers.experimental.preprocessing.RandomZoom(0.1),
])
 batch_size = 32
AUTOTUNE = -1
def prepare(ds, shuffle=False, augment=False):
x = resize_and_rescale(x)
 if shuffle:
ds = ds.shuffle(1000)
 ds = ds.batch(batch_size)
 if augment:
ds = data_augmentation(x, training=True)
 return ds.prefetch(-1 )<set_options>
last_amount = test.loc[(test['Country/Region']=='Italy')&(test['Date']=='2020-03-24'),'ConfirmedCases_x']
datagen = ImageDataGenerator(rescale=1/255.,
zoom_range = 0.3,
height_shift_range = 0.3,
width_shift_range = 0.3,
rotation_range = 25,
shear_range=0.5,
fill_mode="constant",
cval=0 
)
 valid_datagen = ImageDataGenerator(rescale=1./255)
 test_datagen = ImageDataGenerator(rescale=1./255 )<train_model>
last_fat = test.loc[(test['Country/Region']=='Italy')&(test['Date']=='2020-03-24'),'Fatalities_x']
start_time = time.time()
model, history = do_module_CNN(x_train, y_train, x_valid, y_valid)
 print("--- Model Train: %s seconds ---" %(time.time() - start_time))
summarize_diagnostics(history)
 print("Final loss: {0:.4f}, final accuracy: {1:.4f}".format(final_loss, final_acc))<predict_on_test>
i = 0
k = 35
start_time = time.time()
y_hat = model.predict_generator(test_datagen.flow(x_test, shuffle=False))
y_pred = np.argmax(y_hat,axis=1)
 print("--- Model Predict: %s seconds ---" %(time.time() - start_time))
print("y shape", y_pred.shape )<save_to_csv>
for date in dates:
k = k-1
i = i + 1
test.loc[(test['Country/Region']=='Italy')&(test['Date']==date),'ConfirmedCases_x'] = last_amount.values[0]+i*(5000-(100*i))
test.loc[(test['Country/Region']=='Italy')&(test['Date']==date),'Fatalities_x'] = last_fat.values[0]+i*(800-(10*i))
pd_submit=pd.DataFrame({"id": list(range(len(y_pred))),"label": y_pred})
 pd_submit.to_csv('submission.csv',index=False,header=True )<count_values>
test.loc[(test['Country/Region']=='Italy')]
pd_submit['label'].value_counts()<set_options>
sub = test[['ForecastId', 'ConfirmedCases_x','Fatalities_x']]
sns.set(rc={'figure.figsize':(11.7,5.27)})
<load_from_csv>
sub.columns = ['ForecastId', 'ConfirmedCases', 'Fatalities']
df_train_raw = pd.read_csv(os.path.join('/kaggle/input/Kannada-MNIST/train.csv'))
df_valid_raw = pd.read_csv(os.path.join('/kaggle/input/Kannada-MNIST/Dig-MNIST.csv'))
df_test_raw = pd.read_csv(os.path.join('/kaggle/input/Kannada-MNIST/test.csv'))<concatenate>
sub.loc[sub['ConfirmedCases']<0, 'ConfirmedCases'] = 0
df_train_raw=pd.concat([df_train_raw,df_valid_raw])
 df_train_raw.shape<prepare_x_and_y>
sub.loc[sub['Fatalities']<0, 'Fatalities'] = 0
image_x=28
image_y=28<data_type_conversions>
sub.to_csv('submission.csv',index=False )