Kaggle competition: COVID19 Global Forecasting (Week 1)
train = pd.read_csv(TRAIN_DATA_FILE)
 test = pd.read_csv(TEST_DATA_FILE)
 test_l = pd.read_csv(TEST_LABELS_FILE )<count_missing_values>
train = pd.read_csv('.. /input/covid19-global-forecasting-week-1/train.csv')
 test = pd.read_csv('.. /input/covid19-global-forecasting-week-1/test.csv' )
train.isnull().any() ,test.isnull().any()<categorify>
train['date_datetime'] = train['Date'].apply(lambda x:(datetime.datetime.strptime(x, '%Y-%m-%d')) )
def clean_text(text):
text = text.lower()
text = re.sub(r"what's", "what is ", text)
 text = re.sub(r"'s", " ", text)
 text = re.sub(r"'ve", " have ", text)
 text = re.sub(r"can't", "cannot ", text)
 text = re.sub(r"n't", " not ", text)
 text = re.sub(r"i'm", "i am ", text)
 text = re.sub(r"'re", " are ", text)
 text = re.sub(r"'d", " would ", text)
 text = re.sub(r"'ll", " will ", text)
 text = re.sub(r"'scuse", " excuse ", text)
 text = re.sub('\W', ' ', text)
 text = re.sub('\s+', ' ', text)
 text = re.sub(r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}', ' _ip_ ', text)
 text = re.sub(r'(['"\.\(\)\!\?\-\\\/\,])', r' \1 ',text)
 text = re.sub(r'([\;\:\|•«
])', ' ', text)
 text = text.replace('&', ' and ')
 text = text.replace('@', ' at ')
 text = text.replace('0', ' zero ')
 text = text.replace('1', ' one ')
 text = text.replace('2', ' two ')
 text = text.replace('3', ' three ')
 text = text.replace('4', ' four ')
 text = text.replace('5', ' five ')
 text = text.replace('6', ' six ')
 text = text.replace('7', ' seven ')
 text = text.replace('8', ' eight ')
 text = text.replace('9', ' nine ')
 text = text.strip(' ')
 return text<feature_engineering>
def days_convert(date):
return(date - datetime.datetime(2020,1,22)).days
train['days_since_start'] = train['date_datetime'].apply(days_convert )
train['comment_text'] = train['comment_text'].map(lambda com : clean_text(com))<prepare_x_and_y>
def rename_countries(country):
if country == 'US':
country = 'United States'
elif country == 'Gambia, The' or country == 'The Gambia':
country = "Gambia"
elif country == 'The Bahamas':
country = 'Bahamas'
elif country == 'Taiwan*':
country = 'Taiwan'
elif country == 'Republic of the Congo' or country == 'Congo(Kinshasa)' or country == 'Congo(Brazzaville)':
country = 'DR Congo'
elif country == 'Korea, South':
country = 'South Korea'
elif country == 'Czechia':
country = 'Czech Republic'
return country
train['Country/Region'] = train['Country/Region'].apply(rename_countries )
list_sentences_train = train["comment_text"].fillna("_na_" ).values
list_classes = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]
y = train[list_classes].values
list_sentences_test = test["comment_text"].fillna("_na_" ).values<string_transform>
def clean_name(country):
country = country.split('[')[0]
return country
population['country'] = population['Country or area'].apply(clean_name)
 population['pop'] = population['Population(1 July 2019)']
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(list_sentences_train))
list_tokenized_train = tokenizer.texts_to_sequences(list_sentences_train)
 list_tokenized_test = tokenizer.texts_to_sequences(list_sentences_test)
 X_t = pad_sequences(list_tokenized_train, maxlen=maxlen)
 X_te = pad_sequences(list_tokenized_test, maxlen=maxlen )<feature_engineering>
def get_population(country_name):
try:
return population[population['country'] == country_name]['pop'].reset_index() ['pop'][0]
except:
if country_name == 'Taiwan':
return 23_510_000
elif country_name == 'Reunion':
return 859_959
elif country_name == 'Kosovo':
return 1_880_000
elif country_name == 'Jersey':
return 106_000
elif country_name == 'Holy See':
return 1_000
elif country_name == 'Guernsey':
return 66_300
elif country_name == 'Cruise Ship':
return 3_700
elif country_name == "Cote d'Ivoire":
return 22_700_000
else:
print(country_name)
 train['population'] = train['Country/Region'].apply(get_population )
<feature_engineering>
def change_countries(country):
if country == 'DR Congo':
country = 'Democratic Republic of the Congo'
elif country == 'Bahamas':
country = 'The Bahamas'
return country
train['Country/Region2'] = train['Country/Region'].apply(change_countries)
 def get_median(country):
try:
return age[age['Country/Territory'] == country]['Median(Years)'].reset_index() ['Median(Years)'][0]
except:
if country == 'Cruise Ship':
return 53
elif country == 'Eswatini':
return 20.5
elif country == 'French Guiana':
return 24.8
elif country == 'Gambia':
return 20.7
elif country == 'Guadeloupe':
return 35
elif country == 'Holy See':
return 40
elif country == 'Martinique':
return 45.4
elif country == 'Mayotte':
return 20.1
elif country == 'Reunion':
return 45
print(country)
 train['median_age'] = train['Country/Region2'].apply(get_median )
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector<choose_model_class>
life = pd.read_csv('/kaggle/input/world-bank-data-1960-to-2016/life_expectancy.csv')[['Country Name','2016']]
life.head()
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(CuDNNLSTM(50, return_sequences=True))(x)
 x = Conv1D(64, kernel_size = 3, padding = "valid", kernel_initializer = "glorot_uniform" )(x)
 avg_pool = GlobalAvgPool1D()(x)
 max_pool = GlobalMaxPool1D()(x)
 conc = concatenate([avg_pool,max_pool])
 x = GlobalMaxPool1D()(x)
 x = Dense(50, activation='elu' )(x)
 x = Dropout(0.25 )(x)
 x = Dense(6, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'] )<import_modules>
def change_countries(country):
if country == 'DR Congo':
country = 'Democratic Republic of the Congo'
elif country == 'Bahamas':
country = 'The Bahamas'
return country
def get_expectancy(country):
try:
return life[life['Country Name'] == country]['2016'].reset_index() ['2016'][0]
except:
if country == 'Brunei':
return 77
elif country == 'Democratic Republic of the Congo':
return 60
elif country == 'Cruise Ship':
return 85
elif country == 'Egypt':
return 72.7
elif country == 'Eswatini':
return 42
elif country == 'French Guiana':
return 75.9
elif country == 'Gambia':
return 64.9
elif country == 'Guadeloupe':
return 81.84
elif country == 'Guernsey':
return 82.5
elif country == 'Holy See':
return 78
elif country == 'Iran':
return 71.4
elif country == 'Jersey':
return 81.9
elif country == 'South Korea':
return 82.7
elif country == 'Kyrgyzstan':
return 71.4
elif country == 'Martinique':
return 79.6
elif country == 'Mayotte':
return 76.83
elif country == 'North Macedonia':
return 76
elif country == 'Reunion':
return 76.4
elif country == 'Russia':
return 70.8
elif country == 'Saint Lucia':
return 77.8
elif country == 'Saint Vincent and the Grenadines':
return 75.3
elif country == 'Slovakia':
return 77.1
elif country == 'Taiwan':
return 80.2
elif country == 'The Bahamas':
return 72.4
elif country == 'Gambia':
return 64.9
elif country == 'Venezuela':
return 75.8
print(country)
 train['life_expectancy'] = train['Country/Region2'].apply(get_expectancy )
import tensorflow as tf
from keras.models import Sequential
from keras.layers import Flatten, Dense, Dropout, Activation, LSTM, Bidirectional, GlobalMaxPool1D, Embedding,AveragePooling1D
from keras import Input, Model
from keras import layers, initializers, regularizers, constraints, optimizers
from keras.callbacks import EarlyStopping
from keras.callbacks import ModelCheckpoint<choose_model_class>
for index in train[train['Country/Region']=='Andorra']['life_expectancy'].index:
train.loc[index,'life_expectancy'] = 82.8
for index in train[train['Country/Region']=='Greenland']['life_expectancy'].index:
train.loc[index,'life_expectancy'] = 72.4
for index in train[train['Country/Region']=='Monaco']['life_expectancy'].index:
train.loc[index,'life_expectancy'] = 89.5
for index in train[train['Country/Region']=='San Marino']['life_expectancy'].index:
train.loc[index,'life_expectancy'] = 85.6
file_path = "bi_lstm.best.hdf5"
checkpoint = ModelCheckpoint(file_path, monitor='val_loss',verbose=1, save_best_only=True, mode = 'min')
 early = EarlyStopping(monitor="val_loss", mode="min",patience=1 )<train_model>
train[train['life_expectancy'].isna() ==True]
batch_size = 32
epochs = 10
callbacks_list = [checkpoint,early]
history = model.fit(X_t,y, batch_size=batch_size, epochs=epochs, validation_split=0.1, callbacks=callbacks_list )<set_options>
X = train[['Lat','Long','days_since_start','population','median_age']]
y = train['ConfirmedCases']
X_train,X_test,y_train,y_test = train_test_split(X,y,test_size=0.3 )
history.history<save_to_csv>
rf = RandomForestClassifier()
rf.fit(X_train,y_train )
y_test = model.predict([X_te], batch_size=1024, verbose=1)
 sample_submission = pd.read_csv(f'{path}{comp}sample_submission.csv.zip')
 sample_submission[list_classes] = y_test
sample_submission.to_csv('submission.csv', index=False )<train_model>
mean_absolute_error(rf.predict(X_test),y_test )
print("end" )<install_modules>
X1 = train[['Lat','Long','days_since_start','population','median_age','life_expectancy']]
y1 = train['Fatalities']
X_train1,X_test1,y_train1,y_test1 = train_test_split(X1,y1,test_size=0.3 )
!pip install transformers<define_variables>
rf1 = RandomForestClassifier()
rf1.fit(X_train1,y_train1 )
dataset_directory = '.. /input/jigsaw-toxic-comment-classification-challenge'<load_from_csv>
mean_absolute_error(rf1.predict(X_test1),y_test1 )
!mkdir data
!unzip {dataset_directory}/train.csv.zip -d data/
!unzip {dataset_directory}/test.csv.zip -d data/
!unzip {dataset_directory}/test_labels.csv.zip -d data/
!unzip {dataset_directory}/sample_submission.csv.zip -d data/<import_modules>
test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-1/test.csv')
 test.head()
from tqdm.notebook import tqdm
import numpy as np
import pandas as pd
import tensorflow as tf<define_variables>
test['date_datetime'] = test['Date'].apply(lambda x:(datetime.datetime.strptime(x, '%Y-%m-%d')))
 def days_convert(date):
return(date - datetime.datetime(2020,1,22)).days
test['days_since_start'] = test['date_datetime'].apply(days_convert)
 test['Country/Region'] = test['Country/Region'].apply(rename_countries)
 test['population'] = test['Country/Region'].apply(get_population)
 def change_countries(country):
if country == 'DR Congo':
country = 'Democratic Republic of the Congo'
elif country == 'Bahamas':
country = 'The Bahamas'
return country
test['Country/Region2'] = test['Country/Region'].apply(change_countries)
 test['median_age'] = test['Country/Region2'].apply(get_median)
 test['life_expectancy'] = test['Country/Region2'].apply(get_expectancy)
 for index in test[test['Country/Region']=='Andorra']['life_expectancy'].index:
test.loc[index,'life_expectancy'] = 82.8
for index in test[test['Country/Region']=='Greenland']['life_expectancy'].index:
test.loc[index,'life_expectancy'] = 72.4
for index in test[test['Country/Region']=='Monaco']['life_expectancy'].index:
test.loc[index,'life_expectancy'] = 89.5
for index in test[test['Country/Region']=='San Marino']['life_expectancy'].index:
test.loc[index,'life_expectancy'] = 85.6
test.head()
train_path = 'data/train.csv'
test_path = 'data/test.csv'
test_labels_path = 'data/test_labels.csv'
subm_path = 'data/sample_submission.csv'<load_from_csv>
predictions = rf.predict(test[['Lat','Long','days_since_start','population','median_age']])
 predictions1 = rf1.predict(test[['Lat','Long','days_since_start','population','median_age','life_expectancy']] )
label_cols = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']
df_train = pd.read_csv(train_path)
 df_test = pd.read_csv(test_path)
 df_test_labels = pd.read_csv(test_labels_path)
 df_test_labels = df_test_labels.set_index('id')
 df_train.head()<categorify>
submit = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-1/submission.csv')
 submit['ConfirmedCases'] = predictions
submit['ConfirmedCases'] = submit['ConfirmedCases'].apply(int)
 submit['Fatalities'] = predictions1
submit['Fatalities'] = submit['Fatalities'].apply(int)
 submit.head()
bert_model_name = 'bert-base-uncased'
tokenizer = BertTokenizer.from_pretrained(bert_model_name, do_lower_case=True)
 MAX_LEN = 128
def tokenize_sentences(sentences, tokenizer, max_seq_len = 128):
tokenized_sentences = []
for sentence in tqdm(sentences):
tokenized_sentence = tokenizer.encode(
 sentence,
add_special_tokens = True,
max_length = max_seq_len, 
)
 tokenized_sentences.append(tokenized_sentence)
 return tokenized_sentences
def create_attention_masks(tokenized_and_padded_sentences):
attention_masks = []
for sentence in tokenized_and_padded_sentences:
att_mask = [int(token_id > 0)for token_id in sentence]
attention_masks.append(att_mask)
 return np.asarray(attention_masks)
 input_ids = tokenize_sentences(df_train['comment_text'], tokenizer, MAX_LEN)
 input_ids = pad_sequences(input_ids, maxlen=MAX_LEN, dtype="long", value=0, truncating="post", padding="post")
 attention_masks = create_attention_masks(input_ids )<split>
submit.to_csv('submission.csv',index=False )
<prepare_x_and_y><EOS> 
test.to_csv('test.csv')
 train.to_csv('train.csv' )