Kaggle competition: COVID19 Global Forecasting (Week 2)
validation_examples['polarity1'] = validation_examples['project_essay_1'].apply(get_polarity)
 validation_examples['subjectivity1'] = validation_examples['project_essay_1'].apply(get_subjectivity)
 validation_examples['polarity2'] = validation_examples['project_essay_2'].apply(get_polarity)
 validation_examples['subjectivity2'] = validation_examples['project_essay_2'].apply(get_subjectivity )<feature_engineering>
pd.set_option('display.max_columns',500)
 filterwarnings('ignore' )
test_examples['polarity1'] = test_examples['project_essay_1'].apply(get_polarity)
 test_examples['subjectivity1'] = test_examples['project_essay_1'].apply(get_subjectivity)
 test_examples['polarity2'] = test_examples['project_essay_2'].apply(get_polarity)
 test_examples['subjectivity2'] = test_examples['project_essay_2'].apply(get_subjectivity )<drop_column>
train = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-2/train.csv', parse_dates=['Date'])
 test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-2/test.csv', parse_dates=['Date'] )
to_drop = ['project_subject_categories', 'project_subject_subcategories', 'project_title', 'project_essay_1', 'project_essay_2', 'project_resource_summary']
for col in to_drop:
training_examples.drop([col], axis=1, inplace=True)
 validation_examples.drop([col], axis=1, inplace=True)
 test_examples.drop([col], axis=1, inplace=True )<compute_test_metric>
train = train.rename(columns={'Country_Region':'Country','Province_State':'State'})
 test = test.rename(columns={'Country_Region':'Country','Province_State':'State'} )
training_features = csr_matrix(hstack([training_examples.values, train_project_subject_categories, train_project_subject_subcategories, train_project_title, train_project_resource_summary, train_project_essay_1, train_project_essay_2]))
validation_features = csr_matrix(hstack([validation_examples.values, validation_project_subject_categories, validation_project_subject_subcategories, validation_project_title, validation_project_resource_summary, validation_project_essay_1, validation_project_essay_2]))
test_features = csr_matrix(hstack([test_examples.values, test_project_subject_categories, test_project_subject_subcategories, test_project_title, test_project_resource_summary, test_project_essay_1, test_project_essay_2]))<train_model>
train.State = train.State.fillna('Empty')
 test.State = test.State.fillna('Empty' )
<init_hyperparams>
train['day'] = train['Date'].dt.day
train['month'] = train['Date'].dt.month
train['dayofweek'] = train['Date'].dt.dayofweek
train['dayofyear'] = train['Date'].dt.dayofyear
train['quarter'] = train['Date'].dt.quarter
train['weekofyear'] = train['Date'].dt.weekofyear
test['day'] = test['Date'].dt.day
test['month'] = test['Date'].dt.month
test['dayofweek'] = test['Date'].dt.dayofweek
test['dayofyear'] = test['Date'].dt.dayofyear
test['quarter'] = test['Date'].dt.quarter
test['weekofyear'] = test['Date'].dt.weekofyear
params = {
'boosting_type': 'gbdt',
'objective': 'binary',
'metric': 'auc',
'max_depth': 16,
'num_leaves': 31,
'learning_rate': 0.025,
'feature_fraction': 0.85,
'bagging_fraction': 0.85,
'bagging_freq': 5,
'verbose': 0,
'num_threads': 1,
'lambda_l2': 1,
'min_gain_to_split': 0,
}
model2 = lgb.train(
 params,
lgb.Dataset(training_features, training_targets),
num_boost_round=10000,
valid_sets=[lgb.Dataset(validation_features, validation_targets)],
verbose_eval=100,
early_stopping_rounds=100
)<save_to_csv>
from xgboost.sklearn import XGBRegressor
submission = pd.DataFrame()
submission['id'] = test['id']
submission['project_is_approved'] = model2.predict(test_features, num_iteration=model2.best_iteration)
 submission.to_csv('submission.csv', index=False )<load_pretrained>
model = XGBRegressor(n_estimators=1000 )
nltk.download("punkt" )<string_transform>
countries = train.Country.unique().tolist()
results_df = pd.DataFrame({'ForecastId': [], 'ConfirmedCases': [], 'Fatalities': []})
 for c in countries:
states = train.loc[train.Country == c,:].State.unique().tolist()
for state in states:
X_train = train.loc[(train.Country == c)&(train.State == state), :]
y1 = X_train[['ConfirmedCases']]
y2 = X_train[['Fatalities']]
X_train = X_train.drop(['Id','ConfirmedCases','Fatalities','State','Country','Date'], axis=1)
 X_test = test.loc[(test.Country == c)&(test.State == state), :]
results_temp = X_test[['ForecastId']]
X_test = X_test.drop(['ForecastId','State','Country','Date'], axis=1)
 model_confirmed = model.fit(X_train, y1)
 prediction_confirmed = model_confirmed.predict(X_test)
 model_fatalities = model.fit(X_train, y2)
 prediction_fatalities = model_fatalities.predict(X_test)
 results_temp['ConfirmedCases'] = prediction_confirmed
results_temp['Fatalities'] = prediction_fatalities
results_df = pd.concat([results_df, results_temp], axis=0)
def tokenize_sentences(sentences, words_dict):
tokenized_sentences = []
for sentence in tqdm.tqdm(sentences):
if hasattr(sentence, "decode"):
sentence = sentence.decode("utf-8")
 tokens = nltk.tokenize.word_tokenize(sentence)
 result = []
for word in tokens:
word = word.lower()
if word not in words_dict:
words_dict[word] = len(words_dict)
 word_index = words_dict[word]
result.append(word_index)
 tokenized_sentences.append(result)
 return tokenized_sentences, words_dict<categorify>
sub = results_df.copy()
def read_embedding_list(file_path):
embedding_word_dict = {}
embedding_list = []
f = open(file_path)
 for index, line in enumerate(f):
if index == 0:
continue
values = line.split()
word = values[0]
try:
coefs = np.asarray(values[1:], dtype='float32')
 except:
continue
embedding_list.append(coefs)
 embedding_word_dict[word] = len(embedding_word_dict)
 f.close()
embedding_list = np.array(embedding_list)
 return embedding_list, embedding_word_dict<drop_column>
sub.ConfirmedCases = sub.ConfirmedCases.astype(int)
 sub.Fatalities = sub.Fatalities.astype(int)
 sub.ForecastId = sub.ForecastId.astype(int )
def clear_embedding_list(embedding_list, embedding_word_dict, words_dict):
cleared_embedding_list = []
cleared_embedding_word_dict = {}
for word in words_dict:
if word not in embedding_word_dict:
continue
word_id = embedding_word_dict[word]
row = embedding_list[word_id]
cleared_embedding_list.append(row)
 cleared_embedding_word_dict[word] = len(cleared_embedding_word_dict)
 return cleared_embedding_list, cleared_embedding_word_dict<categorify>
sub.to_csv('submission.csv', index=False )
def convert_tokens_to_ids(tokenized_sentences, words_list, embedding_word_dict, sentences_length):
words_train = []
for sentence in tokenized_sentences:
current_words = []
for word_index in sentence:
word = words_list[word_index]
word_id = embedding_word_dict.get(word, len(embedding_word_dict)- 2)
 current_words.append(word_id)
 if len(current_words)>= sentences_length:
current_words = current_words[:sentences_length]
else:
current_words += [len(embedding_word_dict)- 1] *(sentences_length - len(current_words))
words_train.append(current_words)
 return words_train<import_modules>
sub.to_csv('submission.csv', index=False )