Kaggle competition: Competição DSA de Machine Learning
lasso = make_pipeline(RobustScaler() , Lasso(alpha =0.0001))
lasso.fit(X, y)
 r2_score(lasso.predict(X), y )<train_model>
warnings.filterwarnings('ignore' )
ENet = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0001, l1_ratio=.9))
ENet.fit(X, y)
 r2_score(ENet.predict(X), y )<train_model>
df_train = pd.read_csv('.. /input/dataset_treino.csv', sep = ',', encoding = 'utf-8')
 df_test = pd.read_csv('.. /input/dataset_teste.csv', sep = ',', encoding = 'utf-8' )
stacking = StackingRegressor(estimators=[ 
('lgbm', lgbm_grid.best_estimator_), 
('xgb', xgb_grid.best_estimator_), 
('lasso', lasso.steps[1][1]), 
('elastic',ENet.steps[1][1])],
final_estimator=lgbm_grid.best_estimator_,
passthrough=True)
 model = stacking.fit(X, y)
 r2_score(model.predict(X), y )<prepare_output>
df_train[df_train.isnull().any(axis=1)]
submission = pd.DataFrame()
submission['Id'] = test_df['Id']
submission['SalePrice'] = np.round(list(model.predict(test)) , 0)
 submission.head()<save_to_csv>
df_train.isnull().values.any()
submission.to_csv("submission.csv", index=False )<save_to_csv>
df_train.isnull().any()
submission.to_csv("submission.csv", index=False )<load_from_csv>
df_train.isnull().sum()
gender_submission_path = "/kaggle/input/titanic/gender_submission.csv"
train_path = "/kaggle/input/titanic/train.csv"
test_path = "/kaggle/input/titanic/test.csv"
gender_submission = pd.read_csv(gender_submission_path)
 train_data = pd.read_csv(train_path)
 test_data = pd.read_csv(test_path )<count_missing_values>
df_test.isnull().sum()
rate_cabin_Nan = 1-(train_data.Cabin.count() /len(train_data.Cabin))
print("% of NaN values: " ,rate_cabin_Nan )<data_type_conversions>
def info_variavel(df):
print(df.head())
 print('')
 print('Quantidade de registros:', df.count())
 print('Possui valores missing?:', df.isnull().any())
 print('Quantidade de valores missing:', df.isnull().sum())
 print('')
 print('Valores únicos:')
 print(df.unique())
 print('')
 print('Contagem dos elementos:')
 print(df.value_counts() )
combine = [train_data, test_data]
for dataset in combine:
dataset["Sex"] = dataset["Sex"].map({"female": 1, "male" : 0} ).astype(int)
 train_data.head()<set_options>
index = 'Order'
frequent_port = train_data.Embarked.dropna().mode() [0]
frequent_port<categorify>
del df_train[index]
del df_test['OrderId']
for dataset in combine:
dataset["Embarked"] = dataset["Embarked"].map({"S" : 0, "C" : 1, "Q" : 2} ).astype(int)
 train_data.head()<prepare_x_and_y>
index = 'Property Id'
y = train_data.Survived
features = ["Pclass", "Sex", "Age", "SibSp", "Parch", "Fare", "Embarked"]
X = train_data[features]
X_test = test_data[features]<save_to_csv>
del df_train[index]
model = RandomForestClassifier(n_estimators=100, max_depth=5, random_state=1)
 model.fit(imputed_X, y)
 predictions = model.predict(imputed_X_test)
 output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': predictions})
 output.to_csv('my_submission.csv', index=False)
 print("Your submission was successfully saved!" )<set_options>
index = 'Property Name'
%config InlineBackend.figure_format = 'retina'
%matplotlib inline<load_from_csv>
del df_train[index]
del df_test[index]
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )<load_from_csv>
index = 'Parent Property Id'
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )<concatenate>
del df_train[index]
del df_test[index]
all_data = pd.concat(( train.loc[:,'MSSubClass':'SaleCondition'],
test.loc[:,'MSSubClass':'SaleCondition']))<feature_engineering>
index = 'Parent Property Name'
train["SalePrice"] = np.log1p(train["SalePrice"])
 numeric_feats = all_data.dtypes[all_data.dtypes != "object"].index
skewed_feats = train[numeric_feats].apply(lambda x: skew(x.dropna()))
skewed_feats = skewed_feats[skewed_feats > 0.75]
skewed_feats = skewed_feats.index
all_data[skewed_feats] = np.log1p(all_data[skewed_feats] )<categorify>
del df_train[index]
del df_test[index]
all_data = pd.get_dummies(all_data )<data_type_conversions>
index = 'BBL - 10 digits'
all_data = all_data.fillna(all_data.mean() )<prepare_x_and_y>
del df_train[index]
del df_test[index]
X_train = all_data[:train.shape[0]]
X_test = all_data[train.shape[0]:]
y = train.SalePrice<compute_train_metric>
index = 'NYC Borough, Block and Lot(BBL)self-reported'
def rmse_cv(model):
rmse= np.sqrt(-cross_val_score(model, X_train, y, scoring="neg_mean_squared_error", cv = 5))
return(rmse )<choose_model_class>
del df_train[index]
del df_test[index]
model_ridge = Ridge()<define_search_space>
index = 'NYC Building Identification Number(BIN)'
alphas = [0.05, 0.1, 0.3, 1, 3, 5, 10, 15, 30, 50, 75]
cv_ridge = [rmse_cv(Ridge(alpha = alpha)).mean()
for alpha in alphas]<train_on_grid>
del df_train[index]
del df_test[index]
model_lasso = LassoCV(alphas = [1, 0.1, 0.001, 0.0005] ).fit(X_train, y )<find_best_params>
index = 'Address 1(self-reported)'
rmse_cv(model_lasso ).mean()<define_variables>
del df_train[index]
del df_test[index]
print("Lasso picked " + str(sum(coef != 0)) + " variables and eliminated the other " + str(sum(coef == 0)) + " variables" )<concatenate>
index = 'Address 2'
imp_coef = pd.concat([coef.sort_values().head(10),
coef.sort_values().tail(10)] )<import_modules>
del df_train[index]
del df_test[index]
import xgboost as xgb<train_on_grid>
index = 'Postal Code'
dtrain = xgb.DMatrix(X_train, label = y)
 dtest = xgb.DMatrix(X_test)
 params = {"max_depth":2, "eta":0.1}
model = xgb.cv(params, dtrain, num_boost_round=500, early_stopping_rounds=100 )<train_model>
del df_train[index]
del df_test[index]
model_xgb = xgb.XGBRegressor(n_estimators=360, max_depth=2, learning_rate=0.1)
 model_xgb.fit(X_train, y )<predict_on_test>
index = 'Street Number'
xgb_preds = np.expm1(model_xgb.predict(X_test))
lasso_preds = np.expm1(model_lasso.predict(X_test))<create_dataframe>
del df_train[index]
del df_test[index]
predictions = pd.DataFrame({"xgb":xgb_preds, "lasso":lasso_preds})
 predictions.plot(x = "xgb", y = "lasso", kind = "scatter" )<prepare_output>
index = 'Street Name'
preds = 0.7*lasso_preds + 0.3*xgb_preds<save_to_csv>
del df_train[index]
del df_test[index]
solution = pd.DataFrame({"id":test.Id, "SalePrice":preds})
 solution.to_csv("ridge_sol.csv", index = False )<import_modules>
index = 'Borough'
from keras.layers import Dense
from keras.models import Sequential
from keras.regularizers import l1
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split<normalization>
df_train[index] = df_train[index].astype('category' ).cat.codes
df_test[index] = df_test[index].astype('category' ).cat.codes
X_train = StandardScaler().fit_transform(X_train )<split>
df_train = df_train[df_train[index] > -1]
X_tr, X_val, y_tr, y_val = train_test_split(X_train, y, random_state = 3 )<choose_model_class>
intervalo_valores(df_train[index] )
model = Sequential()
model.add(Dense(1, input_dim = X_train.shape[1], W_regularizer=l1(0.001)))
 model.compile(loss = "mse", optimizer = "adam" )<train_model>
index = 'DOF Gross Floor Area'
hist = model.fit(X_tr, y_tr, validation_data =(X_val, y_val))<set_options>
del df_train[index]
del df_test[index]
%matplotlib inline
pd.set_option('max_colwidth',250)
 pd.set_option('max_columns',250)
 pd.set_option('max_rows',500 )<load_from_csv>
index = 'Primary Property Type - Self Selected'
train = pd.read_csv('.. /input/duth-dbirlab2-1/train.csv')
 test = pd.read_csv('.. /input/duth-dbirlab2-1/test.csv' )<categorify>
df_train[index] = df_train[index].astype('category' ).cat.codes
df_test[index] = df_test[index].astype('category' ).cat.codes
for df in [train,test]:
for c in df.drop(['obs_id'],axis=1):
if(df[c].dtype=='object'):
lbl = LabelEncoder()
lbl.fit(list(df[c].values))
df[c] = lbl.transform(list(df[c].values))<define_variables>
intervalo_valores(df_train[index] )
ntrain = train.shape[0]
ntest = test.shape[0]
SEED = 11
NFOLDS = 5
folds = KFold(n_splits= NFOLDS, random_state=SEED, shuffle=True )<prepare_x_and_y>
index = 'List of All Property Use Types at Property'
cols_to_exclude = ['obs_id','Overall Probability']
df_train_columns = [c for c in train.columns if c not in cols_to_exclude]
y_train = train['Overall Probability'].ravel()
x_train = train[df_train_columns].values
x_test = test[df_train_columns].values<split>
df_train[index] = df_train[index].astype('category' ).cat.codes
df_test[index] = df_test[index].astype('category' ).cat.codes
def train_model(X_train, X_test, Y_train, folds=5, model_type='lgb',plot_feature_importance=True):
oof = np.zeros(ntrain)
 prediction = np.zeros(ntest)
 scores = []
feature_importance = pd.DataFrame()
for fold_n,(train_index, valid_index)in enumerate(folds.split(X_train,Y_train)) :
print('Fold', fold_n+1, 'started at', time.ctime())
 x_train, x_valid = X_train[train_index], X_train[valid_index]
y_train, y_valid = Y_train[train_index], Y_train[valid_index]
if model_type == 'ridge':
model = linear_model.Ridge(alpha=.5)
 model.fit(x_train, y_train)
 y_pred_valid = model.predict(x_valid)
 y_pred = model.predict(X_test)
 if model_type == 'linear':
model = LinearRegression()
model.fit(x_train, y_train)
 y_pred_valid = model.predict(x_valid)
 y_pred = model.predict(X_test)
 if model_type == 'rf':
model = RandomForestRegressor(min_weight_fraction_leaf=0.05,n_jobs=-2,random_state=0, max_depth=4, n_estimators=100)
 model.fit(x_train, y_train)
 y_pred_valid = model.predict(x_valid)
 y_pred = model.predict(X_test)
 if model_type == 'lgb':
lgb_params = {
'num_leaves': 7,
'min_data_in_leaf': 20,
'min_sum_hessian_in_leaf': 11,
'objective': 'regression',
'max_depth': 6,
'learning_rate': 0.05,
'boosting': "gbdt",
'feature_fraction': 0.8,
'feature_fraction_seed': 9,
'max_bin ': 55,
"bagging_freq": 5,
"bagging_fraction": 0.8,
"bagging_seed": 9,
'metric': 'rmse',
'lambda_l1': 0.1,
'verbosity': -1,
'min_child_weight': 5.34,
'reg_alpha': 1.130,
'reg_lambda': 0.360,
'subsample': 0.8,
}
model = lgb.LGBMRegressor(**lgb_params, n_estimators = 20000, n_jobs = -1)
 model.fit(x_train, y_train, eval_set=[(x_train, y_train),(x_valid, y_valid)], eval_metric='rmse',verbose=10000, early_stopping_rounds=100)
 y_pred_valid = model.predict(x_valid)
 y_pred_valid = np.clip(y_pred_valid, a_min=0, a_max=1)
 y_pred = model.predict(X_test, num_iteration=model.best_iteration_)
 y_pred = np.clip(y_pred, a_min=0, a_max=1)
 fold_importance = pd.DataFrame()
fold_importance["feature"] = train[df_train_columns].columns
fold_importance["importance"] = model.feature_importances_
fold_importance["fold"] = fold_n + 1
feature_importance = pd.concat([feature_importance, fold_importance], axis=0)
 oof[valid_index] = y_pred_valid.reshape(-1,)
 scores.append(mean_squared_error(y_valid, y_pred_valid)** 0.5)
 prediction += y_pred
if(model_type == 'lgb' and plot_feature_importance==True):
cols = feature_importance[["feature", "importance"]].groupby("feature" ).mean().sort_values(
 by="importance", ascending=False)[:50].index
best_features = feature_importance.loc[feature_importance.feature.isin(cols)]
plt.figure(figsize=(16, 12)) ;
sns.barplot(x="importance", y="feature", data=best_features.sort_values(by="importance", ascending=False)) ;
plt.title('LGB Features(avg over folds)')
 prediction /= NFOLDS
print('CV mean score: {0:.5f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)))
 return oof, prediction<train_model>
intervalo_valores(df_train[index] )
oof, prediction = train_model(X_train=x_train, X_test=x_test, Y_train=y_train, folds=folds, model_type='rf', plot_feature_importance=True )<save_to_csv>
index = 'Largest Property Use Type'
sample_submission = pd.read_csv('.. /input/duth-dbirlab2-1/sample_submission.csv')
 sub_df = pd.DataFrame({"obs_id":sample_submission["obs_id"].values})
 sub_df["Overall Probability"] = prediction
sub_df["Overall Probability"] = sub_df["Overall Probability"].apply(lambda x: 1 if x>1 else 0 if x<0 else x)
 sub_df.to_csv("submission.csv", index=False )<import_modules>
df_train[index] = df_train[index].astype('category' ).cat.codes
df_test[index] = df_test[index].astype('category' ).cat.codes
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import os<load_from_csv>
intervalo_valores(df_train[index] )
submission1 = pd.read_csv('/kaggle/input/tfidf/submission(27 ).csv')
 submission2 = pd.read_csv('.. /input/tpuinference-super-fast-xlmroberta/submission(47 ).csv' )<feature_engineering>
index = 'Largest Property Use Type - Gross Floor Area(ft²)'
s = []
s1 = submission1['toxic'].tolist()
s2 = submission2['toxic'].tolist()
for i in range(len(s1)) :
s.append(( s1[i]+s2[i])/2)
 submission1['toxic'] = s<save_to_csv>
intervalo_valores(df_train[index] )
submission1.to_csv('submission.csv', index=False )<define_variables>
index = '2nd Largest Property Use Type'
sub_path = ".. /input/blending"
all_files = os.listdir(sub_path)
 all_files<load_from_csv>
df_train[index] = df_train[index].astype('category' ).cat.codes
df_test[index] = df_test[index].astype('category' ).cat.codes
outs = [pd.read_csv(os.path.join(sub_path, f), index_col=0)for f in all_files]
concat_sub = pd.concat(outs, axis=1)
 cols = list(map(lambda x: "jigsaw" + str(x), range(len(concat_sub.columns))))
concat_sub.columns = cols
concat_sub.reset_index(inplace=True)
 concat_sub.head()
ncol = concat_sub.shape[1]<feature_engineering>
intervalo_valores(df_train[index] )
concat_sub['jigsaw_max'] = concat_sub.iloc[:, 1:ncol].max(axis=1)
 concat_sub['jigsaw_min'] = concat_sub.iloc[:, 1:ncol].min(axis=1)
 concat_sub['jigsaw_mean'] = concat_sub.iloc[:, 1:ncol].mean(axis=1)
 concat_sub['jigsaw_median'] = concat_sub.iloc[:, 1:ncol].median(axis=1 )<define_variables>
index = '2nd Largest Property Use - Gross Floor Area(ft²)'
cutoff_lo = 0.7
cutoff_hi = 0.3<save_to_csv>
df_train[index] = df_train[index].str.replace('Not Available', '0')
 df_test[index] = df_test[index].str.replace('Not Available', '0' )
concat_sub['toxic'] = concat_sub['jigsaw_mean']
concat_sub[['toxic']].to_csv('submission2.csv',
index=False, float_format='%.6f' )<save_to_csv>
df_train[index] = df_train[index].astype('float64')
 df_test[index] = df_test[index].astype('float64' )
concat_sub['toxic'] = concat_sub['jigsaw_median']
concat_sub[['toxic']].to_csv('submission1.csv',
index=False, float_format='%.6f' )<save_to_csv>
intervalo_valores(df_train[index] )
concat_sub['toxic'] = np.where(np.all(concat_sub.iloc[:,1:ncol] > cutoff_lo, axis=1), 1,
np.where(np.all(concat_sub.iloc[:,1:ncol] < cutoff_hi, axis=1),
0, concat_sub['jigsaw_median']))
concat_sub[['toxic']].to_csv('submission3.csv',
index=False, float_format='%.6f' )<feature_engineering>
index = '3rd Largest Property Use Type'
concat_sub['toxic'] = np.where(np.all(concat_sub.iloc[:,1:ncol] > cutoff_lo, axis=1),
concat_sub['jigsaw_max'],
np.where(np.all(concat_sub.iloc[:,1:ncol] < cutoff_hi, axis=1),
concat_sub['jigsaw_min'],
concat_sub['jigsaw_mean']))
concat_sub[['toxic']].to_csv('submission4.csv',
index=False, float_format='%.6f' )<feature_engineering>
df_train[index] = df_train[index].astype('category' ).cat.codes
df_test[index] = df_test[index].astype('category' ).cat.codes
concat_sub['toxic'] = np.where(np.all(concat_sub.iloc[:,1:ncol] > cutoff_lo, axis=1),
concat_sub['jigsaw_max'],
np.where(np.all(concat_sub.iloc[:,1:ncol] < cutoff_hi, axis=1),
concat_sub['jigsaw_min'],
concat_sub['jigsaw_median']))
concat_sub[['toxic']].to_csv('submission5.csv',
index=False, float_format='%.6f' )<set_options>
intervalo_valores(df_train[index] )
%matplotlib inline<load_from_csv>
index = '3rd Largest Property Use Type - Gross Floor Area(ft²)'
train = pd.read_csv('.. /input/digit-recognizer/train.csv')
 test = pd.read_csv('.. /input/digit-recognizer/test.csv' )<prepare_x_and_y>
df_train[index] = df_train[index].str.replace('Not Available', '0')
 df_test[index] = df_test[index].str.replace('Not Available', '0' )
X = train.drop('label', axis = 1)
 y = train['label']
del train<feature_engineering>
df_train[index] = df_train[index].astype('float64')
 df_test[index] = df_test[index].astype('float64' )
X = X / 255.0
test = test / 255.0<categorify>
intervalo_valores(df_train[index] )
labels = pd.get_dummies(y)
 labels = labels.values<split>
index = 'Year Built'
X_train, X_val, y_train, y_val = train_test_split(X, labels, test_size = 0.1,random_state=42 )<import_modules>
intervalo_valores(df_train[index] )
from keras.preprocessing.image import ImageDataGenerator<train_model>
index = 'Number of Buildings - Self-reported'
datagen = ImageDataGenerator(shear_range= 0.2, zoom_range= 0.2)
 datagen.fit(X_train )<import_modules>
intervalo_valores(df_train[index] )
from keras.layers import Dropout, Conv2D, MaxPool2D, Dense, Flatten
from keras.models import Sequential<choose_model_class>
index = 'Occupancy'
model = Sequential()
model.add(Conv2D(filters = 64, kernel_size = 5,activation ='relu', input_shape =(28,28,1)))
 model.add(Conv2D(filters = 64, kernel_size = 5,activation ='relu'))
model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Dropout(0.2))
model.add(Conv2D(filters = 32, kernel_size = 3,activation ='relu'))
model.add(Conv2D(filters = 32, kernel_size = 3,activation ='relu'))
model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Dropout(0.2))
model.add(Flatten())
 model.add(Dense(256, activation = "relu"))
model.add(Dropout(0.2))
model.add(Dense(10, activation = "softmax"))<choose_model_class>
intervalo_valores(df_train[index] )
model.compile(optimizer= 'adam', loss= 'categorical_crossentropy', metrics= ['accuracy'] )<train_model>
index = 'Metered Areas(Energy)'
performance = model.fit_generator(datagen.flow(X_train,y_train, batch_size=64), epochs = 30,
validation_data =(X_val,y_val), verbose = 3 )<import_modules>
df_train[index] = df_train[index].astype('category' ).cat.codes
df_test[index] = df_test[index].astype('category' ).cat.codes
from sklearn.metrics import confusion_matrix, classification_report<predict_on_test>
intervalo_valores(df_train[index] )
pred = model.predict(X_val )<prepare_output>
index = 'Metered Areas(Water)'
pred_classes = np.argmax(pred,axis = 1)
 y_true = np.argmax(y_val,axis = 1 )<compute_test_metric>
df_train[index] = df_train[index].astype('category' ).cat.codes
df_test[index] = df_test[index].astype('category' ).cat.codes
print(classification_report(y_true, pred_classes))<predict_on_test>
intervalo_valores(df_train[index] )
test_preds = model.predict(test)
 test_preds = np.argmax(test_preds,axis = 1)
 test_preds = pd.Series(test_preds,name="Label" )<save_to_csv>
index = 'Site EUI(kBtu/ft²)'
submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),test_preds],axis = 1)
 submission.to_csv("cnn_mnist.csv",index=False )<compute_test_metric>
intervalo_valores(df_train[index] )
def score(result, children_wishes, gifts_wishes):
children_count = len(children_wishes)
 unique_gifts_count = len(gifts_wishes)
 gifts_wishes_len = len(gifts_wishes[0])
 child_wishes_len = len(children_wishes[0])
 gifts_of_each_type_count = children_count // unique_gifts_count
child_happiness = np.full(( children_count, unique_gifts_count), -1, dtype=np.int16)
 array = np.arange(child_wishes_len, 0, -1)* 2 + 1
for c in range(children_count):
child_happiness[c, children_wishes[c]] += array
gift_happiness = np.full(( children_count, unique_gifts_count), -1, dtype=np.int16)
 array = np.arange(gifts_wishes_len, 0, -1)* 2 + 1
for g in range(unique_gifts_count):
gift_happiness[gifts_wishes[g], g] += array
child_sum = 0
gift_sum = 0
for i, gift_id in enumerate(result):
child_sum = child_sum + child_happiness[i, gift_id]
gift_sum = gift_sum + gift_happiness[i, gift_id]
denominator2 = children_count * child_wishes_len * 2
denominator1 = unique_gifts_count * gifts_of_each_type_count * gifts_wishes_len * 2
anch3 =(child_sum / denominator2)** 3
ansh3 =(gift_sum / denominator1)** 3
return ansh3 + anch3 
<load_pretrained>
index = 'Weather Normalized Site EUI(kBtu/ft²)'
class AbstractDataReader(ABC):
@abstractmethod
def read_children_wishes_and_gift_wishes(self):
pass
class DataFromFileReader(AbstractDataReader):
def __init__(self, child_wishlist_path, gift_wishlist_path):
self.child_wishlist_path = child_wishlist_path
self.gift_wishlist_path = gift_wishlist_path
def read_children_wishes_and_gift_wishes(self):
return self._read_data_from_csv()
def _read_data_from_csv(self):
children_wishes = pd.read_csv(self.child_wishlist_path, header=None)
 children_wishes.drop([0], axis=1, inplace=True)
 gift_wishes = pd.read_csv(self.gift_wishlist_path, header=None)
 gift_wishes.drop([0], axis=1, inplace=True)
 return children_wishes.values, gift_wishes.values
class DataRandomReader(AbstractDataReader):
def __init__(self, children_count, gift_count, children_wishes_len, gifts_wishes_len):
self.children_num = children_count
self.gift_num = gift_count
self.children_wishes_len = children_wishes_len
self.gifts_wishes_len = gifts_wishes_len
def read_children_wishes_and_gift_wishes(self):
return self._generate_random_wishes(self.children_num, self.children_wishes_len, self.gift_num), \
self._generate_random_wishes(self.gift_num, self.gifts_wishes_len, self.children_num)
 @staticmethod
def _generate_random_wishes(row_num, columns_num, max_num_in_column_exclusive):
result = np.empty(( row_num, columns_num), dtype=np.int32)
 for i in range(row_num):
result[i, :] = np.random.choice(max_num_in_column_exclusive, size=columns_num, replace=False)
 return result
class Solver:
def __init__(self, children_wishes, gifts_wishes, p=1, d=1):
self.children_wishes = children_wishes
self.gifts_wishes = gifts_wishes
self.p = p
self.d = d
self.result = None
self.happinesses = None
self.graph = None
self.unhappy_children = None
self.children_count = len(children_wishes)
 self.unique_gifts_count = len(gifts_wishes)
 twins = math.ceil(0.04 * self.children_count / 2.) * 2
triplets = math.ceil(
 0.005 * self.children_count / 3.) * 3
self.last_triplet_id = triplets - 1
self.last_twin_id = triplets + twins - 1
self.gifts_of_each_type_count = int(self.children_count // self.unique_gifts_count)
 @property
def gift_wishes_len(self):
return len(self.gifts_wishes[0])
 @property
def child_wishes_len(self):
return len(self.children_wishes[0])
 @property
def vent_index(self):
return self.children_count + self.unique_gifts_count
def solve(self):
self.prepare_happinesses()
self.graph = self.create_flow_graph()
if not DEBUG:
del self.happinesses
gc.collect()
self.graph.Solve()
gc.collect()
optimal_cost = self.graph.OptimalCost()
print('koszt optymalny: ', optimal_cost)
 print('przepływ', self.graph.MaximumFlow())
 self.calculate_result_from_graph()
del self.graph
gc.collect()
def calculate_result_from_graph(self):
result = np.full(self.children_count, fill_value=-1, dtype=np.int32)
 graph = self.graph
for i in range(graph.NumArcs()):
if graph.Flow(i)== 0:
continue
if graph.Tail(i)< self.children_count:
flow = graph.Flow(i)
 while flow > 0:
j = 0
while result[graph.Tail(i)+ j] != -1:
j = j + 1
flow = flow - 1
result[graph.Tail(i)+ j] = self.node_index_to_gift_id(graph.Head(i))
self.result = result
def prepare_happinesses(self):
print("Tworzenie słownika szczęść")
 happinesses = dict()
for gift_id, gift_wishes in enumerate(self.gifts_wishes):
for child_position_in_gift_wishes, child_id in enumerate(gift_wishes):
h = self.d *(self.gift_wishes_len - child_position_in_gift_wishes - 1)
 child_happinesses = happinesses.setdefault(child_id, dict())
 current = child_happinesses.get(gift_id, -1)
 happinesses[child_id][gift_id] = current + h
for child_id, child_wishes in enumerate(self.children_wishes):
for gift_position_in_child_wishes, gift_id in enumerate(child_wishes):
h = self.p *(self.child_wishes_len - gift_position_in_child_wishes - 1)
 child_happinesses = happinesses.setdefault(child_id, dict())
 if gift_id in child_happinesses:
current = child_happinesses[gift_id] + 1
else:
current = -1
happinesses[child_id][gift_id] = current + h
for x in happinesses.values() :
for y in x.keys() :
x[y] += 2
self.happinesses = happinesses
print("Stworzono słownika szczęść")
 print("-" * 20)
 def gift_id_to_node_index(self, gift_id):
return self.children_count + gift_id
def node_index_to_gift_id(self, node_index):
return node_index - self.children_count
def create_flow_graph(self):
print("Tworze graf")
 graph = pywrapgraph.SimpleMinCostFlow()
print('-' * 5 + "trojaczki")
 for i in tqdm(range(0, self.last_triplet_id, 3)) :
graph.SetNodeSupply(i, 3)
 graph.AddArcWithCapacityAndUnitCost(tail=i, head=self.vent_index, capacity=3, unit_cost=0)
 positive_gift_ids = set(self.happinesses[i].keys())| set(self.happinesses[i + 1].keys())| set(
 self.happinesses[i + 2].keys())
 for gift_id in positive_gift_ids:
gift_id = int(gift_id)
 first_happiness = self.happinesses[i].get(gift_id, 0)
 second_happiness = self.happinesses[i + 1].get(gift_id, 0)
 third_happiness = self.happinesses[i + 2].get(gift_id, 0)
 avg_happ = int(mean([first_happiness, second_happiness, third_happiness]))
if avg_happ > 0:
gift_node_index = self.gift_id_to_node_index(gift_id)
 graph.AddArcWithCapacityAndUnitCost(tail=i, head=gift_node_index, capacity=3, unit_cost=-avg_happ)
 print('-' * 5 + "dwojaczki")
 for i in tqdm(range(self.last_triplet_id + 1, self.last_twin_id, 2)) :
graph.SetNodeSupply(i, 2)
 graph.AddArcWithCapacityAndUnitCost(tail=i, head=self.vent_index, capacity=2, unit_cost=0)
 positive_gift_ids = set(self.happinesses[i].keys())| set(self.happinesses[i + 1].keys())
 for gift_id in positive_gift_ids:
gift_id = int(gift_id)
 first_happiness = self.happinesses[i].get(gift_id, 0)
 second_happiness = self.happinesses[i + 1].get(gift_id, 0)
 avg_happ = int(mean([first_happiness, second_happiness]))
if avg_happ > 0:
gift_node_index = self.gift_id_to_node_index(gift_id)
 graph.AddArcWithCapacityAndUnitCost(tail=i, head=gift_node_index, capacity=2, unit_cost=-avg_happ)
 print('-' * 5 + "jedynaki")
 for i in tqdm(range(self.last_twin_id + 1, self.children_count)) :
graph.SetNodeSupply(i, 1)
 graph.AddArcWithCapacityAndUnitCost(tail=i, head=self.vent_index, capacity=1, unit_cost=0)
 for gift_id, happ in self.happinesses[i].items() :
gift_node_index = self.gift_id_to_node_index(gift_id)
 graph.AddArcWithCapacityAndUnitCost(tail=i, head=int(gift_node_index), capacity=1, unit_cost=-happ)
 for i in range(self.unique_gifts_count):
gift_node_index = self.gift_id_to_node_index(i)
 graph.AddArcWithCapacityAndUnitCost(tail=gift_node_index, head=self.vent_index,
capacity=self.gifts_of_each_type_count, unit_cost=0)
 graph.SetNodeSupply(self.vent_index, -self.children_count)
 print("Stworzono graf")
 print('-' * 20)
 return graph
DEBUG = False
ROBERT = True
MACIEK = not ROBERT
if DEBUG:
if ROBERT:
INPUT_FOLDER = '/home/robert/Downloads/'
elif MACIEK:
INPUT_FOLDER = 'input/'
else:
INPUT_FOLDER = '/kaggle/input/santa-gift-matching/'
CHILD_WISHLIST_PATH = INPUT_FOLDER + 'child_wishlist_v2.csv.zip'
GIFT_WISHLIST_PATH = INPUT_FOLDER + 'gift_goodkids_v2.csv.zip' 
<load_pretrained>
df_train[index] = df_train[index].str.replace('Not Available', '0')
 df_test[index] = df_test[index].str.replace('Not Available', '0' )
print("Wczytuje dane")
 if DEBUG:
reader = DataRandomReader(children_count=100_000, gift_count=100, children_wishes_len=1, gifts_wishes_len=1)
 else:
reader = DataFromFileReader(CHILD_WISHLIST_PATH, GIFT_WISHLIST_PATH)
 c_wishes, g_wishes = reader.read_children_wishes_and_gift_wishes()
print("Wczytano dane")
 print('-' * 20)
 if DEBUG:
print("children wishes: ")
 print(c_wishes)
 print("gifts wishes: ")
 print(g_wishes)
 p = 1_000_000
solver = Solver(c_wishes, g_wishes, p, d=1)
 solver.solve() 
<save_to_csv>
df_train[index] = df_train[index].astype('float64')
 df_test[index] = df_test[index].astype('float64' )
if not DEBUG:
out = open('temp.csv', 'w')
 out.write('ChildId,GiftId
')
 for gift_id, child_id in enumerate(solver.result):
out.write(str(gift_id)+ ',' + str(child_id)+ '
')
 out.close()<create_dataframe>
intervalo_valores(df_train[index] )
result_copy = result.copy()<prepare_output>
index = 'Weather Normalized Site Electricity Intensity(kWh/ft²)'
result = result_copy<define_variables>
df_train[index] = df_train[index].str.replace('Not Available', '0')
 df_test[index] = df_test[index].str.replace('Not Available', '0' )
if DEBUG:
print("result before fixing")
 print(result)
 print('vent_index', vent_index)
 print('result.max', result.max())
 matches_for_every_gift = np.zeros(unique_gifts_count + 1, dtype=np.int32)
 print(result.max())
 for gift_id in result:
matches_for_every_gift[gift_id] += 1 
<sort_values>
df_train[index] = df_train[index].astype('float64')
 df_test[index] = df_test[index].astype('float64' )
matches = [[matches_count,gift_id] for gift_id,matches_count in enumerate(matches_for_every_gift)]
matches.sort()
print(matches[:10] )<feature_engineering>
intervalo_valores(df_train[index] )
for kid_id, gift_id in enumerate(result):
if gift_id == unique_gifts_count:
for match in matches:
count, not_matched_id = match
if count < 1000:
result[kid_id] = not_matched_id
matches[-1][0] -= 1
match[0] += 1
break 
<find_best_params>
index = 'Weather Normalized Site Natural Gas Intensity(therms/ft²)'
matches_for_every_gift = np.zeros(unique_gifts_count, dtype=np.int32)
 print("result max", result.max())
 for gift_id in result:
matches_for_every_gift[gift_id] += 1 
<create_dataframe>
df_train[index] = df_train[index].str.replace('Not Available', '0')
 df_test[index] = df_test[index].str.replace('Not Available', '0' )
result_copy2 = result.copy()<prepare_output>
df_train[index] = df_train[index].astype('float64')
 df_test[index] = df_test[index].astype('float64' )
result = result_copy2<define_variables>
intervalo_valores(df_train[index] )
if DEBUG:
print("matches_for_every_gift po przydzieleniu każdemu dziecku prezentu")
 print(matches_for_every_gift)
 print(result)
 counter = 0
for child_id in range(0, last_triplet_id, 3):
gift1 = result[child_id]
gift2 = result[child_id + 1]
gift3 = result[child_id + 2]
if gift1 == gift2 == gift3:
continue
counter += 1
if gift1 != gift2 and gift1 != gift3 and gift2 != gift3:
counter += 1
if gift2 == gift3:
dominant = gift3
else:
dominant = gift1
result[child_id] = dominant
result[child_id + 1] = dominant
result[child_id + 2] = dominant
print("zabrane prezenty tylko przez trojaczki", counter)
 for child_id in range(last_triplet_id + 1, last_twin_id, 2):
if result[child_id + 1] == result[child_id]:
continue
counter += 1
result[child_id + 1] = result[child_id]
print("zabrane prezenty razem", counter )<feature_engineering>
index = 'Weather Normalized Source EUI(kBtu/ft²)'
unhappy_children = []
for child_id in range(last_twin_id + 1, len(result)) :
if matches_for_every_gift[result[child_id]] > gifts_of_each_type_count:
unhappy_children.append(child_id)
 old_gift = result[child_id]
least_matched_gift = np.argmin(matches_for_every_gift)
 result[child_id] = least_matched_gift
matches_for_every_gift[old_gift] -= 1
matches_for_every_gift[least_matched_gift] += 1
print("Nieszczęśliwe dzieci", len(unhappy_children))
print("matches_for_every_gift po naprawie")
 print(matches_for_every_gift)
 if matches_for_every_gift.max(initial=-1)> gifts_of_each_type_count:
print('Nie udało sie dopasować prezentów')
 gc.collect()<save_to_csv>
df_train[index] = df_train[index].str.replace('Not Available', '0')
 df_test[index] = df_test[index].str.replace('Not Available', '0' )
if not DEBUG:
out = open('temp2.csv', 'w')
 out.write('ChildId,GiftId
')
 for gift_id, child_id in enumerate(result):
out.write(str(gift_id)+ ',' + str(child_id)+ '
')
 out.close()<categorify>
df_train[index] = df_train[index].astype('float64')
 df_test[index] = df_test[index].astype('float64' )
print("Tworzenie słownika szczęść")
 happinesses = dict()
children_wishes, gifts_wishes = reader.read_children_wishes_and_gift_wishes()
gift_wishes_len = len(gifts_wishes[0])
 child_wishes_len = len(children_wishes[0])
 for gift_id, gift_wishes in enumerate(gifts_wishes):
for child_position_in_gift_wishes, child_id in enumerate(gift_wishes):
h =(gift_wishes_len - child_position_in_gift_wishes - 1)
 child_happinesses = happinesses.setdefault(child_id, dict())
 current = child_happinesses.get(gift_id, -1)
 happinesses[child_id][gift_id] = current + h
for child_id, child_wishes in enumerate(children_wishes):
for gift_position_in_child_wishes, gift_id in enumerate(child_wishes):
h = p *(child_wishes_len - gift_position_in_child_wishes - 1)
 child_happinesses = happinesses.setdefault(child_id, dict())
 if gift_id in child_happinesses:
current = child_happinesses[gift_id] + 1
else:
current = -1
happinesses[child_id][gift_id] = current + h
for x in happinesses.values() :
for y in x.keys() :
x[y] += 2
print("Stworzono słownik szczęść" )<categorify>
intervalo_valores(df_train[index] )
print("Zaczynam heurstykę")
 print("Ilość nieszczęśliwych dzieci:", len(unhappy_children))
for child1_id in range(len(unhappy_children)) :
for child2_id in range(child1_id + 1, len(unhappy_children)) :
child1 = unhappy_children[child1_id]
child2 = unhappy_children[child2_id]
gift1 = result[child1]
gift2 = result[child2]
old_happ1 = happinesses.get(child1, {} ).get(gift1, 0)
 old_happ2 = happinesses.get(child2, {} ).get(gift2, 0)
 new_happ1 = happinesses.get(child1, {} ).get(gift2, 0)
 new_happ2 = happinesses.get(child2, {} ).get(gift1, 0)
 old = old_happ1 + old_happ2
new = new_happ1 + new_happ2
if new > old:
result[child1] = gift2
result[child2] = gift1
print('Skończyłem heurystykę' )<drop_column>
index = 'Fuel Oil
del happinesses<save_to_csv>
df_train[index] = df_train[index].str.replace('Not Available', '0')
 df_test[index] = df_test[index].str.replace('Not Available', '0' )
gc.collect()
s = score(result, c_wishes, g_wishes)
 print("Score: ", s)
 if not DEBUG:
out = open(f'out_{s}.csv', 'w')
 out.write('ChildId,GiftId
')
 for gift_id, child_id in enumerate(result):
out.write(str(gift_id)+ ',' + str(child_id)+ '
')
 out.close()<import_modules>
df_train[index] = df_train[index].astype('float64')
 df_test[index] = df_test[index].astype('float64' )
import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
import seaborn as sns
from sklearn.preprocessing import PowerTransformer<load_from_csv>
intervalo_valores(df_train[index] )
train = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/train.csv")
 test_x = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/test.csv")
 train.columns<prepare_x_and_y>
index = 'Fuel Oil
train_x = train.drop("SalePrice",axis=1)
 train_y = train["SalePrice"]<concatenate>
df_train[index] = df_train[index].str.replace('Not Available', '0')
 df_test[index] = df_test[index].str.replace('Not Available', '0' )
all_data = pd.concat([train_x,test_x],axis=0,sort=True )<drop_column>
df_train[index] = df_train[index].astype('float64')
 df_test[index] = df_test[index].astype('float64' )
train_ID = train['Id']
test_ID = test_x['Id']
all_data.drop("Id", axis = 1, inplace = True )<sort_values>
intervalo_valores(df_train[index] )
all_data_na = all_data.isnull().sum() [all_data.isnull().sum() >0].sort_values(ascending=False)
 all_data_na<sort_values>
index = 'Fuel Oil
na_col_list = all_data.isnull().sum() [all_data.isnull().sum() >0].index.tolist()
all_data[na_col_list].dtypes.sort_values()<data_type_conversions>
df_train[index] = df_train[index].str.replace('Not Available', '0')
 df_test[index] = df_test[index].str.replace('Not Available', '0' )
all_data['LotFrontage'] = all_data.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.median()))
float_list = all_data[na_col_list].dtypes[all_data[na_col_list].dtypes == "float64"].index.tolist()
obj_list = all_data[na_col_list].dtypes[all_data[na_col_list].dtypes == "object"].index.tolist()
all_data[float_list] = all_data[float_list].fillna(0)
 all_data[obj_list] = all_data[obj_list].fillna("None")
 all_data.isnull().sum() [all_data.isnull().sum() > 0]<data_type_conversions>
df_train[index] = df_train[index].astype('float64')
 df_test[index] = df_test[index].astype('float64' )
all_data['MSSubClass'] = all_data['MSSubClass'].apply(str)
 all_data['YrSold'] = all_data['YrSold'].astype(str)
 all_data['MoSold'] = all_data['MoSold'].astype(str )<filter>
intervalo_valores(df_train[index] )
skewed_feats_over = skewed_feats[abs(skewed_feats)> 0.5].index
for i in skewed_feats_over:
print(min(all_data[i]))<feature_engineering>
index = 'Fuel Oil
all_data["FeetPerRoom"] = all_data["TotalSF"]/all_data["TotRmsAbvGrd"]
all_data['YearBuiltAndRemod']=all_data['YearBuilt']+all_data['YearRemodAdd']
all_data['Total_Bathrooms'] =(all_data['FullBath'] +(0.5 * all_data['HalfBath'])+
all_data['BsmtFullBath'] +(0.5 * all_data['BsmtHalfBath']))
all_data['Total_porch_sf'] =(all_data['OpenPorchSF'] + all_data['3SsnPorch'] +
all_data['EnclosedPorch'] + all_data['ScreenPorch'] +
all_data['WoodDeckSF'])
 all_data['haspool'] = all_data['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 all_data['has2ndfloor'] = all_data['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 all_data['hasgarage'] = all_data['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 all_data['hasbsmt'] = all_data['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 all_data['hasfireplace'] = all_data['Fireplaces'].apply(lambda x: 1 if x > 0 else 0 )<count_values>
df_train[index] = df_train[index].str.replace('Not Available', '0')
 df_test[index] = df_test[index].str.replace('Not Available', '0' )
all_data.dtypes.value_counts()<categorify>
df_train[index] = df_train[index].astype('float64')
 df_test[index] = df_test[index].astype('float64' )
all_data = pd.get_dummies(all_data,columns=cal_list)
 all_data.shape<import_modules>
intervalo_valores(df_train[index] )
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.model_selection import KFold, cross_val_score, train_test_split
from sklearn.metrics import mean_squared_error
import xgboost as xgb
from sklearn.model_selection import GridSearchCV
import lightgbm as lgb<split>
index = 'Diesel
train_x, valid_x, train_y, valid_y = train_test_split(
 train_x,
train_y,
test_size=0.15,
random_state=0 )<train_model>
df_train[index] = df_train[index].str.replace('Not Available', '0')
 df_test[index] = df_test[index].str.replace('Not Available', '0' )
dtrain = xgb.DMatrix(train_x, label=train_y)
 dvalid = xgb.DMatrix(valid_x,label=valid_y)
 num_round = 5000
evallist = [(dvalid, 'eval'),(dtrain, 'train')]
evals_result = {}
param = {
'max_depth': 3,
'eta': 0.01,
'objective': 'reg:squarederror',
}
bst = xgb.train(
 param, dtrain,
num_round,
evallist,
evals_result=evals_result,
early_stopping_rounds=1000
)<save_to_csv>
df_train[index] = df_train[index].astype('float64')
 df_test[index] = df_test[index].astype('float64' )
dtest = xgb.DMatrix(test_x)
 my_submission = pd.DataFrame()
my_submission["Id"] = test_ID
my_submission["SalePrice"] = np.exp(bst.predict(dtest))
my_submission.to_csv('submission.csv', index=False )<import_modules>
intervalo_valores(df_train[index] )
import pandas as pd
import numpy as np
from xgboost import XGBRegressor
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder<load_from_csv>
index = 'District Steam Use(kBtu)'
df = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-2/train.csv')
 df_test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-2/test.csv')
 submission = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-2/submission.csv' )<categorify>
df_train[index] = df_train[index].str.replace('Not Available', '0')
 df_test[index] = df_test[index].str.replace('Not Available', '0' )
def fill_States(s):
if pd.isna(s['Province_State']):
if pd.isna(s['Country_Region']):
return "NA"
else:
return s['Country_Region']
else:
return s['Province_State']<feature_engineering>
df_train[index] = df_train[index].astype('float64')
 df_test[index] = df_test[index].astype('float64' )
df['Province_State'] = df.apply(fill_States,axis=1 )<feature_engineering>
intervalo_valores(df_train[index] )
df_test['Province_State'] = df_test.apply(fill_States,axis=1 )<data_type_conversions>
index = 'Natural Gas Use(kBtu)'
df['Date'] = pd.to_datetime(df['Date'])
 df_test['Date'] = pd.to_datetime(df_test['Date'] )<feature_engineering>
df_train[index] = df_train[index].str.replace('Not Available', '0')
 df_test[index] = df_test[index].str.replace('Not Available', '0' )
df['Date'] = df['Date'].dt.month * 100 + df['Date'].dt.day
df_test['Date'] = df_test['Date'].dt.month * 100 + df_test['Date'].dt.day<create_dataframe>
df_train[index] = df_train[index].astype('float64')
 df_test[index] = df_test[index].astype('float64' )
preds = pd.DataFrame()<feature_engineering>
intervalo_valores(df_train[index] )
preds['ForecastId'] = df_test.ForecastId<categorify>
index = 'Weather Normalized Site Natural Gas Use(therms)'
le = LabelEncoder()
le2 = LabelEncoder()<prepare_x_and_y>
df_train[index] = df_train[index].str.replace('Not Available', '0')
 df_test[index] = df_test[index].str.replace('Not Available', '0' )
for country in df['Country_Region'].unique() :
for state in df[df['Country_Region']==country].loc[:,'Province_State'].unique() :
train = df[ df['Province_State'] == state ].copy()
X = train.drop(['Id','ConfirmedCases','Fatalities'],axis=1)
 X['Province_State'] = le.fit_transform(X['Province_State'])
 X['Country_Region'] = le2.fit_transform(X['Country_Region'])
 y1 = train['ConfirmedCases']
y2 = train['Fatalities']
test = df_test[ df_test['Province_State'] == state ].copy()
X_test = test.drop(['ForecastId'],axis=1)
 X_test['Province_State'] = le.transform(X_test['Province_State'])
 X_test['Country_Region'] = le2.transform(X_test['Country_Region'])
 model1 = XGBRegressor(n_estimators=1000)
 model1.fit(X,y1)
 preds.at[test['ForecastId'] - 1,'ConfirmedCases'] = model1.predict(X_test)
 model2 = XGBRegressor(n_estimators=1000)
 model2.fit(X,y2)
 preds.at[test['ForecastId'] - 1,'Fatalities'] = model2.predict(X_test)
<save_to_csv>
df_train[index] = df_train[index].astype('float64')
 df_test[index] = df_test[index].astype('float64' )
preds.to_csv('submission.csv',index=False )<load_from_csv>
intervalo_valores(df_train[index] )
result_dict = {"filename": [],
"person_id": [],
"id": [],
"xmin": [], "xmax": [],
"ymin": [], "ymax": []}
name_to_id = pd.read_csv("/kaggle/input/asn10e-final-submission-coml-facial-recognition/person_id_mapping.csv")
 print("predicting on img 0.jpg")
 locations, names = predict("0.jpg")
 for name, location in zip(names, locations):
name = " ".join(name.split("_"))
if name != "Unknown":
print(name)
 id_ = name_to_id[name_to_id["Person"] == name].values.tolist() [0][0]
result_dict["filename"].append("0.jpg")
 result_dict["person_id"].append(id_)
 result_dict["id"].append("0_%s" % id_)
 result_dict["xmin"].append(location[3])
 result_dict["xmax"].append(location[1])
 result_dict["ymin"].append(location[2])
 result_dict["ymax"].append(location[0])
<predict_on_test>
index = 'Electricity Use - Grid Purchase(kBtu)'
print("predicting on img 1.jpg")
 locations, names = predict("1.jpg")
 for name, location in zip(names, locations):
name = " ".join(name.split("_"))
print(name)
 if name == "Joshua Lo":
name = "Rung-Chuan Lo"
if name != "Unknown":
id_ = name_to_id[name_to_id["Person"] == name].values.tolist() [0][0]
result_dict["filename"].append("1.jpg")
 result_dict["person_id"].append(id_)
 result_dict["id"].append("1_%s" % id_)
 result_dict["xmin"].append(location[3])
 result_dict["xmax"].append(location[1])
 result_dict["ymin"].append(location[2])
 result_dict["ymax"].append(location[0] )<save_to_csv>
df_train[index] = df_train[index].str.replace('Not Available', '0')
 df_test[index] = df_test[index].str.replace('Not Available', '0' )
print(result_dict)
 with open('submission.csv', 'w', newline='')as file:
writer = csv.writer(file)
 writer.writerow(["filename", "person_id", "id", "xmin", "xmax", "ymin", 'ymax'])
 for i in range(len(result_dict["filename"])) :
row = [result_dict["filename"][i], result_dict["person_id"][i], result_dict["id"][i], result_dict["xmin"][i],
result_dict["xmax"][i], result_dict["ymin"][i], result_dict["ymax"][i]]
writer.writerow(row)
 print(len(result_dict["filename"]))<feature_engineering>
df_train[index] = df_train[index].astype('float64')
 df_test[index] = df_test[index].astype('float64' )
def clean_data(data):
data["Fare"] = data["Fare"].fillna(data["Fare"].dropna().median())
 data["Age"] = data["Age"].fillna(data["Age"].dropna().median())
 data.loc[data["Sex"] == "male", "Sex"] = 0
data.loc[data["Sex"] == "female", "Sex"] = 1
data["Embarked"] = data["Embarked"].fillna("S")
 data.loc[data["Embarked"] == "S", "Embarked"] = 0
data.loc[data["Embarked"] == "C", "Embarked"] = 1
data.loc[data["Embarked"] == "Q", "Embarked"] = 2<set_options>
intervalo_valores(df_train[index] )
warnings.filterwarnings("ignore" )<load_from_csv>
index = 'Weather Normalized Site Electricity(kWh)'
train = pd.read_csv("/kaggle/input/titanic/train.csv")
 clean_data(train)
 target = train['Survived'].values
features = train[['Pclass', 'Age', 'Fare', 'Embarked', 'Sex', 'SibSp', 'Parch']].values
classifier = linear_model.LogisticRegression()
classifier_ = classifier.fit(features, target)
 print(classifier_.score(features, target))<train_model>
df_train[index] = df_train[index].str.replace('Not Available', '0')
 df_test[index] = df_test[index].str.replace('Not Available', '0' )
poly = preprocessing.PolynomialFeatures(degree=2)
 poly_features = poly.fit_transform(features)
 classifier_ = classifier.fit(poly_features, target)
 print(classifier_.score(poly_features, target))<load_from_csv>
df_train[index] = df_train[index].astype('float64')
 df_test[index] = df_test[index].astype('float64' )
train = pd.read_csv("/kaggle/input/titanic/train.csv")
 clean_data(train)
 target = train["Survived"].values
features = train[["Pclass", "Age", "Fare", "Embarked", "Sex", "SibSp", "Parch"]].values
decision_tree = tree.DecisionTreeClassifier(random_state = 42)
 decision_tree_ = decision_tree.fit(features, target)
 print(decision_tree_.score(features, target))<train_on_grid>
intervalo_valores(df_train[index] )
generalized_tree = tree.DecisionTreeClassifier(
 random_state = 1,
max_depth = 7,
min_samples_split = 2)
 generalized_tree_ = generalized_tree.fit(features, target)
 scores = model_selection.cross_val_score(generalized_tree, features, target, scoring = 'accuracy', cv = 50)
 print(scores)
 print(scores.mean() )<save_to_csv>
index = 'Total GHG Emissions(Metric Tons CO2e)'
data = export_graphviz(DecisionTreeClassifier(max_depth=3 ).fit(features, target), out_file=None,
feature_names = ['Pclass', 'Age', 'Fare', 'Embarked', 'Sex', 'SibSp', 'Parch'],
class_names = ['Survived(0)', 'Survived(1)'],
filled = True, rounded = True, special_characters = True)
 graph = graphviz.Source(data)
 graph<train_on_grid>
df_train[index] = df_train[index].str.replace('Not Available', '0')
 df_test[index] = df_test[index].str.replace('Not Available', '0' )
forest = RandomForestClassifier(random_state = 1)
 n_estimators = [1740, 1742, 1745, 1750]
max_depth = [6, 7, 8]
min_samples_split = [4, 5, 6]
min_samples_leaf = [4, 5, 6]
oob_score = ['True']
hyperF = dict(n_estimators = n_estimators, max_depth = max_depth, min_samples_split = min_samples_split, min_samples_leaf = min_samples_leaf, oob_score = oob_score)
 gridF = GridSearchCV(forest, hyperF, verbose = 1, n_jobs = 4)
 bestF = gridF.fit(features, target )<load_from_csv>
df_train[index] = df_train[index].astype('float64')
 df_test[index] = df_test[index].astype('float64' )
train = pd.read_csv("/kaggle/input/titanic/train.csv")
 clean_data(train)
 target = train["Survived"].values
features = train[["Pclass", "Age", "Fare", "Embarked", "Sex", "SibSp", "Parch"]].values
r_forest = RandomForestClassifier(criterion='gini',bootstrap=True,
n_estimators=1745,
max_depth=7,
min_samples_split=6,
min_samples_leaf=6,
max_features='auto',
oob_score=True,
random_state=123,
n_jobs=-1,
verbose=0)
 rf_clf = r_forest.fit(features, target)
 print(rf_clf.score(features, target))<compute_train_metric>
intervalo_valores(df_train[index] )
rf_clf.oob_score_<choose_model_class>
index = 'Direct GHG Emissions(Metric Tons CO2e)'
value = 1.50
width = 0.75
clf1 = LogisticRegression(random_state=0)
 clf2 = RandomForestClassifier(random_state=0)
 clf3 = DecisionTreeClassifier(random_state=0)
 eclf = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], weights=[1, 1, 1], voting='soft')
 X_list = train[["Pclass", "Age", "Fare", "Embarked", "Sex", "SibSp", "Parch"]]
X = np.asarray(X_list, dtype=np.float32)
 y_list = train["Survived"]
y = np.asarray(y_list, dtype=np.int32)
 gs = gridspec.GridSpec(2, 2)
 fig = plt.figure(figsize=(10, 8))
labels = ['Logistic Regression',
'Random Forest',
'Decision Tree',
'Ensemble']
for clf, lab, grd in zip([clf1, clf2, clf3, eclf],
labels,
itertools.product([0, 1],
repeat=2)) :
clf.fit(X, y)
 ax = plt.subplot(gs[grd[0], grd[1]])
 fig = plot_decision_regions(X=X, y=y, clf=clf,
filler_feature_values={2: value, 3: value, 4: value, 5: value, 6: value},
filler_feature_ranges={2: width, 3: width, 4: width, 5: width, 6: width},
legend=2)
 plt.title(lab)
 plt.show()<save_to_csv>
df_train[index] = df_train[index].str.replace('Not Available', '0')
 df_test[index] = df_test[index].str.replace('Not Available', '0' )
test = pd.read_csv("/kaggle/input/titanic/test.csv")
 clean_data(test)
 prediction = rf_clf.predict(test[["Pclass", "Age", "Fare", "Embarked", "Sex", "SibSp", "Parch"]])
 output = pd.DataFrame({'PassengerId': test.PassengerId, 'Survived': prediction})
 output.to_csv('titanic_submission.csv', index=False)
 print("Submission successful" )<feature_engineering>
df_train[index] = df_train[index].astype('float64')
 df_test[index] = df_test[index].astype('float64' )
def clean_data(data):
data["Fare"] = data["Fare"].fillna(data["Fare"].dropna().median())
 data["Age"] = data["Age"].fillna(data["Age"].dropna().median())
 data.loc[data["Sex"] == "male", "Sex"] = 0
data.loc[data["Sex"] == "female", "Sex"] = 1
data["Embarked"] = data["Embarked"].fillna("S")
 data.loc[data["Embarked"] == "S", "Embarked"] = 0
data.loc[data["Embarked"] == "C", "Embarked"] = 1
data.loc[data["Embarked"] == "Q", "Embarked"] = 2<set_options>
intervalo_valores(df_train[index] )
warnings.filterwarnings("ignore" )<load_from_csv>
index = 'Indirect GHG Emissions(Metric Tons CO2e)'
train = pd.read_csv("/kaggle/input/titanic/train.csv")
 clean_data(train)
 target = train['Survived'].values
features = train[['Pclass', 'Age', 'Fare', 'Embarked', 'Sex', 'SibSp', 'Parch']].values
classifier = linear_model.LogisticRegression()
classifier_ = classifier.fit(features, target)
 print(classifier_.score(features, target))<train_model>
df_train[index] = df_train[index].str.replace('Not Available', '0')
 df_test[index] = df_test[index].str.replace('Not Available', '0' )
poly = preprocessing.PolynomialFeatures(degree=2)
 poly_features = poly.fit_transform(features)
 classifier_ = classifier.fit(poly_features, target)
 print(classifier_.score(poly_features, target))<load_from_csv>
df_train[index] = df_train[index].astype('float64')
 df_test[index] = df_test[index].astype('float64' )
train = pd.read_csv("/kaggle/input/titanic/train.csv")
 clean_data(train)
 target = train["Survived"].values
features = train[["Pclass", "Age", "Fare", "Embarked", "Sex", "SibSp", "Parch"]].values
decision_tree = tree.DecisionTreeClassifier(random_state = 42)
 decision_tree_ = decision_tree.fit(features, target)
 print(decision_tree_.score(features, target))<train_on_grid>
intervalo_valores(df_train[index] )
generalized_tree = tree.DecisionTreeClassifier(
 random_state = 1,
max_depth = 7,
min_samples_split = 2)
 generalized_tree_ = generalized_tree.fit(features, target)
 scores = model_selection.cross_val_score(generalized_tree, features, target, scoring = 'accuracy', cv = 50)
 print(scores)
 print(scores.mean() )<save_to_csv>
index = 'Property GFA - Self-Reported(ft²)'
data = export_graphviz(DecisionTreeClassifier(max_depth=3 ).fit(features, target), out_file=None,
feature_names = ['Pclass', 'Age', 'Fare', 'Embarked', 'Sex', 'SibSp', 'Parch'],
class_names = ['Survived(0)', 'Survived(1)'],
filled = True, rounded = True, special_characters = True)
 graph = graphviz.Source(data)
 graph<train_on_grid>
intervalo_valores(df_train[index] )
forest = RandomForestClassifier(random_state = 1)
 n_estimators = [1740, 1742, 1745, 1750]
max_depth = [6, 7, 8]
min_samples_split = [4, 5, 6]
min_samples_leaf = [4, 5, 6]
oob_score = ['True']
hyperF = dict(n_estimators = n_estimators, max_depth = max_depth, min_samples_split = min_samples_split, min_samples_leaf = min_samples_leaf, oob_score = oob_score)
 gridF = GridSearchCV(forest, hyperF, verbose = 1, n_jobs = 4)
 bestF = gridF.fit(features, target )<load_from_csv>
index = 'Water Use(All Water Sources )(kgal)'
train = pd.read_csv("/kaggle/input/titanic/train.csv")
 clean_data(train)
 target = train["Survived"].values
features = train[["Pclass", "Age", "Fare", "Embarked", "Sex", "SibSp", "Parch"]].values
r_forest = RandomForestClassifier(criterion='gini',bootstrap=True,
n_estimators=1745,
max_depth=7,
min_samples_split=6,
min_samples_leaf=6,
max_features='auto',
oob_score=True,
random_state=123,
n_jobs=-1,
verbose=0)
 rf_clf = r_forest.fit(features, target)
 print(rf_clf.score(features, target))<compute_train_metric>
df_train[index] = df_train[index].str.replace('Not Available', '0')
 df_test[index] = df_test[index].str.replace('Not Available', '0' )
rf_clf.oob_score_<choose_model_class>
df_train[index] = df_train[index].astype('float64')
 df_test[index] = df_test[index].astype('float64' )
value = 1.50
width = 0.75
clf1 = LogisticRegression(random_state=0)
 clf2 = RandomForestClassifier(random_state=0)
 clf3 = DecisionTreeClassifier(random_state=0)
 eclf = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], weights=[1, 1, 1], voting='soft')
 X_list = train[["Pclass", "Age", "Fare", "Embarked", "Sex", "SibSp", "Parch"]]
X = np.asarray(X_list, dtype=np.float32)
 y_list = train["Survived"]
y = np.asarray(y_list, dtype=np.int32)
 gs = gridspec.GridSpec(2, 2)
 fig = plt.figure(figsize=(10, 8))
labels = ['Logistic Regression',
'Random Forest',
'Decision Tree',
'Ensemble']
for clf, lab, grd in zip([clf1, clf2, clf3, eclf],
labels,
itertools.product([0, 1],
repeat=2)) :
clf.fit(X, y)
 ax = plt.subplot(gs[grd[0], grd[1]])
 fig = plot_decision_regions(X=X, y=y, clf=clf,
filler_feature_values={2: value, 3: value, 4: value, 5: value, 6: value},
filler_feature_ranges={2: width, 3: width, 4: width, 5: width, 6: width},
legend=2)
 plt.title(lab)
 plt.show()<save_to_csv>
intervalo_valores(df_train[index] )
test = pd.read_csv("/kaggle/input/titanic/test.csv")
 clean_data(test)
 prediction = rf_clf.predict(test[["Pclass", "Age", "Fare", "Embarked", "Sex", "SibSp", "Parch"]])
 output = pd.DataFrame({'PassengerId': test.PassengerId, 'Survived': prediction})
 output.to_csv('titanic_submission.csv', index=False)
 print("Submission successful" )<set_options>
index = 'Water Intensity(All Water Sources )(gal/ft²)'
%matplotlib inline
cf.go_offline()<load_from_csv>
df_train[index] = df_train[index].str.replace('Not Available', '0')
 df_test[index] = df_test[index].str.replace('Not Available', '0' )
train_data=pd.read_csv('/kaggle/input/titanic/train.csv')
 test_data=pd.read_csv('/kaggle/input/titanic/test.csv')
<sort_values>
df_train[index] = df_train[index].astype('float64')
 df_test[index] = df_test[index].astype('float64' )
train_data.isnull().sum().sort_values(ascending = False )<sort_values>
intervalo_valores(df_train[index] )
test_data.isnull().sum().sort_values(ascending = False )<drop_column>
index = 'Source EUI(kBtu/ft²)'
train_data.drop('Cabin',axis=1,inplace=True)
 test_data.drop('Cabin',axis=1,inplace=True )<categorify>
intervalo_valores(df_train[index] )
def impute_age(cols):
Age=cols[0]
Pclass=cols[1]
if pd.isnull(Age):
if Pclass == 1:
return 37
elif Pclass == 2:
return 29
else:
return 24
else:
return Age<feature_engineering>
index = 'Release Date'
train_data['Age'] = train_data[['Age','Pclass']].apply(impute_age,axis=1 )<feature_engineering>
del df_train[index]
del df_test[index]
test_data['Age'] = test_data[['Age','Pclass']].apply(impute_age,axis=1 )<sort_values>
index = 'Water Required?'
train_data.isnull().sum().sort_values(ascending = False )<sort_values>
df_train[index] = df_train[index].astype('category' ).cat.codes
df_test[index] = df_test[index].astype('category' ).cat.codes
test_data.isnull().sum().sort_values(ascending = False )<define_variables>
df_train = df_train[df_train[index] > -1]
list_of_non_numeric_data=list(train_data.select_dtypes(include='object'))
list_of_non_numeric_data<drop_column>
intervalo_valores(df_train[index] )
train_data.drop('Ticket',axis=1,inplace=True)
 test_data.drop('Ticket',axis=1,inplace=True )<string_transform>
index = 'DOF Benchmarking Submission Status'
def getTitles(name):
name = str(name)
 title = name.split('.')[0]
title = title.split(',')
 return title[1]<feature_engineering>
del df_train[index]
del df_test[index]
train_data['Title'] = train_data['Name'].apply(getTitles)
 train_data['Title']<feature_engineering>
index = 'Latitude'
test_data['Title'] = test_data['Name'].apply(getTitles)
 test_data['Title']<drop_column>
del df_train[index]
del df_test[index]
def cleanTitle(title):
if title in [' Mr',' Mrs',' Master',' Miss']:
return title
else:
return "Others"<feature_engineering>
index = 'Longitude'
train_data['Title'] = train_data['Title'].apply(cleanTitle)
 test_data['Title'] = test_data['Title'].apply(cleanTitle )<categorify>
del df_train[index]
del df_test[index]
Title_train = pd.get_dummies(train_data['Title'],drop_first=True)
 Title_test = pd.get_dummies(test_data['Title'],drop_first=True)
 sex_train = pd.get_dummies(train_data['Sex'],drop_first=True)
 embark_train = pd.get_dummies(train_data['Embarked'],drop_first=True)
 sex_test = pd.get_dummies(test_data['Sex'],drop_first=True)
 embark_test = pd.get_dummies(test_data['Embarked'],drop_first=True)
<drop_column>
index = 'Community Board'
train_data.drop(['Sex','Embarked','Name','Title'],axis=1,inplace=True)
 test_data.drop(['Sex','Embarked','Name','Title'],axis=1,inplace=True )<concatenate>
del df_train[index]
del df_test[index]
train_data=pd.concat([train_data,sex_train,embark_train,Title_train],axis=1)
 test_data=pd.concat([test_data,sex_test,embark_test,Title_test],axis=1 )<import_modules>
index = 'Council District'
from sklearn.preprocessing import MinMaxScaler<prepare_x_and_y>
del df_train[index]
del df_test[index]
X_train=train_data.drop(['Survived','PassengerId'],axis=1)
 y_train= train_data['Survived']
X_test=test_data.drop(['PassengerId'],axis=1 )<normalization>
index = 'Census Tract'
Scaler=MinMaxScaler()<normalization>
del df_train[index]
del df_test[index]
X_train = Scaler.fit_transform(X_train)
 X_test = Scaler.transform(X_test )<import_modules>
index = 'NTA'
from sklearn.linear_model import LogisticRegression<train_model>
del df_train[index]
del df_test[index]
logmodel = LogisticRegression(max_iter=10000)
 logmodel.fit(X_train,y_train )<predict_on_test>
index = 'ENERGY STAR Score'
test_data['Survived']=logmodel.predict(X_test )<save_to_csv>
intervalo_valores(df_train[index] )
test_data[['PassengerId', 'Survived']].to_csv('kaggle_submission.csv', index = False )<set_options>
df_train.isnull().any()
init_notebook_mode(connected=True)
 warnings.filterwarnings('ignore' )<load_from_csv>
df_test.isnull().any()
ndf=pd.read_csv('/kaggle/input/covid19-global-forecasting-week-2/test.csv')
 mdfw=pd.read_csv('/kaggle/input/covid19-global-forecasting-week-2/train.csv' )<feature_engineering>
df_train.isnull().sum()
def parser(x):
try:
return datetime.strptime(x,'%Y-%m-%d')
 except:
return datetime.strptime(x,' %Y-%m-%d')
 for i in range(0,len(ndf)) :
ndf.Date[i]=parser(ndf.Date[i])
 for i in range(0,len(mdfw)) :
mdfw.Date[i]=parser(mdfw.Date[i] )<feature_engineering>
df_test.isnull().sum()
pdfw=ndf[ndf['Province_State'].isna() ==True]
countries=list(pdfw.Country_Region.unique())
 mdfw.head()
t1=mdfw[['Country_Region','ConfirmedCases','Date']]
t1.index.name=''
p1=pd.DataFrame(t1[t1['Country_Region']=='Afghanistan'] ).set_index('Date' ).loc['2020-03-28']
p=pd.DataFrame(p1 ).T
i='Afghanistan' 
<feature_engineering>
dataset = df_train.drop(['ENERGY STAR Score'], axis=1)
 dataset['ENERGY STAR Score'] = df_train['ENERGY STAR Score']
for i in countries:
if i not in ['Denmark', 'France', 'Netherlands', 'United Kingdom']:
t1=mdfw[['Country_Region','ConfirmedCases','Date']]
t1.index.name=''
t1=t1[t1['Country_Region']==i]
w1=t1[t1['Date']>=datetime.strptime('2020-03-19', '%Y-%m-%d')]
w1=w1[t1['Date']<=datetime.strptime('2020-03-28', '%Y-%m-%d')].set_index('Date')
 w1.index.name=''
cc1=ndf[ndf['Country_Region']==i].set_index('Date')
 cc=mdfw[mdfw['Country_Region']==i].drop(['Country_Region','Province_State'],axis=1 ).set_index('Date' ).drop('Fatalities',axis=1)
 cc.index.name=''
cc['EverydayCC']=cc.ConfirmedCases.diff()
cc.EverydayCC.iloc[0]=0.0
try:
model = VAR(endog=cc.drop('Id',axis=1))
model_fit = model.fit()
yhat = model_fit.forecast(model_fit.y, steps=33)
 final=pd.DataFrame(data=yhat,index=pd.date_range(start='29/3/2020', periods=33),columns=['ConfirmedCases','EverydayCC'])
 final.drop('ConfirmedCases',axis=1)
 final['ConfirmedCases']=final['EverydayCC'].apply(lambda x: cc.iloc[-1][1]+x)
 final.drop('EverydayCC',axis=1,inplace=True)
 final['Country_Region']=i
w1=w1.append(final)
 w1['Province_State']=cc1['Province_State']
w1['ForecastId']=cc1['ForecastId']
p =p.append(w1)
 except:
print('Exception')
 else:
kkk=mdfw[mdfw['Country_Region']==i ]
lol=kkk[kkk.Province_State.isna() ]
t1=lol[['Country_Region','ConfirmedCases','Date']]
t1.index.name=''
o1=pd.DataFrame(t1[t1['Country_Region']==i] ).set_index('Date')
 o1.index.name=''
t1=o1.loc['2020-03-19':'2020-03-28']
cc2=ndf[ndf['Country_Region']==i].set_index('Date')
 cc1=cc2[cc2['Province_State'].isna() ]
cc=lol[lol['Country_Region']==i].drop(['Country_Region','Province_State'],axis=1 ).set_index('Date' ).drop('Fatalities',axis=1)
 cc.index.name=''
cc['EverydayCC']=cc.ConfirmedCases.diff()
cc.EverydayCC.iloc[0]=0.0
model = VAR(endog=cc.drop('Id',axis=1))
model_fit = model.fit()
yhat = model_fit.forecast(model_fit.y, steps=33)
 final=pd.DataFrame(data=yhat,index=pd.date_range(start='29/3/2020', periods=33),columns=['ConfirmedCases','EverydayCC'])
 final.drop('ConfirmedCases',axis=1)
 final['ConfirmedCases']=final['EverydayCC'].apply(lambda x: cc.iloc[-1][1]+x)
 final.drop('EverydayCC',axis=1,inplace=True)
 final['Country_Region']=i
t1=t1.append(final)
 t1['Country_Region']=cc1['Country_Region']
t1['ForecastId']=cc1['ForecastId']
p =p.append(t1)
<split>
X = dataset.iloc[:,:-1]
y = dataset['ENERGY STAR Score'].values
dp1=p[1:]<create_dataframe>
warnings.filterwarnings('ignore')
 %matplotlib inline
t1=mdfw[['Country_Region','Fatalities','Date']]
t1.index.name=''
t1=pd.DataFrame(t1[t1['Country_Region']=='Afghanistan'].iloc[0] ).T
p=t1.set_index('Date')
<feature_engineering>
from sklearn import linear_model
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.metrics import r2_score
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import MinMaxScaler
import statsmodels.api as sm
for i in countries:
if i not in ['Denmark', 'France', 'Netherlands', 'United Kingdom']:
t1=mdfw[['Country_Region','Fatalities','Date']]
t1.index.name=''
t1=t1[t1['Country_Region']==i]
w1=t1[t1['Date']>=datetime.strptime('2020-03-19', '%Y-%m-%d')]
w1=w1[t1['Date']<=datetime.strptime('2020-03-28', '%Y-%m-%d')].set_index('Date')
 w1.index.name=''
cc1=ndf[ndf['Country_Region']==i].set_index('Date')
 cc=mdfw[mdfw['Country_Region']==i].drop(['Country_Region','Province_State'],axis=1 ).set_index('Date' ).drop('ConfirmedCases',axis=1)
 cc.index.name=''
cc['EverydayCC']=cc.Fatalities.diff()
cc.EverydayCC.iloc[0]=0.0
try:
model = VAR(endog=cc.drop('Id',axis=1))
model_fit = model.fit()
yhat = model_fit.forecast(model_fit.y, steps=33)
 final=pd.DataFrame(data=yhat,index=pd.date_range(start='29/3/2020', periods=33),columns=['Fatalities','EverydayCC'])
 final.drop('Fatalities',axis=1)
 final['Fatalities']=final['EverydayCC'].apply(lambda x: cc.iloc[-1][1]+x)
 final.drop('EverydayCC',axis=1,inplace=True)
 final['Country_Region']=i
w1=w1.append(final)
 w1['Province_State']=cc1['Province_State']
w1['ForecastId']=cc1['ForecastId']
p =p.append(w1)
 except:
print('Exception')
 else:
kkk=mdfw[mdfw['Country_Region']==i ]
lol=kkk[kkk.Province_State.isna() ]
t1=lol[['Country_Region','Fatalities','Date']]
t1.index.name=''
o1=pd.DataFrame(t1[t1['Country_Region']==i] ).set_index('Date')
 o1.index.name=''
t1=o1.loc['2020-03-19':'2020-03-28']
cc2=ndf[ndf['Country_Region']==i].set_index('Date')
 cc1=cc2[cc2['Province_State'].isna() ]
cc=lol[lol['Country_Region']==i].drop(['Country_Region','Province_State'],axis=1 ).set_index('Date' ).drop('ConfirmedCases',axis=1)
 cc.index.name=''
cc['EverydayCC']=cc.Fatalities.diff()
cc.EverydayCC.iloc[0]=0.0
model = VAR(endog=cc.drop('Id',axis=1))
model_fit = model.fit()
yhat = model_fit.forecast(model_fit.y, steps=33)
 final=pd.DataFrame(data=yhat,index=pd.date_range(start='29/3/2020', periods=33),columns=['Fatalities','EverydayCC'])
 final.drop('Fatalities',axis=1)
 final['Fatalities']=final['EverydayCC'].apply(lambda x: cc.iloc[-1][1]+x)
 final.drop('EverydayCC',axis=1,inplace=True)
 final['Country_Region']=i
t1=t1.append(final)
 t1['Country_Region']=cc1['Country_Region']
t1['ForecastId']=cc1['ForecastId']
p =p.append(t1)
<split>
modelo = GradientBoostingRegressor()
dp2=p[1:]<feature_engineering>
Stand_coef_linear_reg.fit(X,y)
 for feature_importances_, var in sorted(zip(map(abs, Stand_coef_linear_reg.steps[1][1].feature_importances_), dataset.columns[:-1]), reverse = True):
print("%6.3f %s" %(feature_importances_,var))
for i in pro:
t1=mdfw[['Province_State','ConfirmedCases','Date']]
t1.index.name=''
t1=t1[t1['Province_State']==i]
w1=t1[t1['Date']>=datetime.strptime('2020-03-19', '%Y-%m-%d')]
w1=w1[t1['Date']<=datetime.strptime('2020-03-28', '%Y-%m-%d')].set_index('Date')
 w1.index.name=''
cc1=ndf[ndf['Province_State']==i].set_index('Date')
 cc=mdfw[mdfw['Province_State']==i].drop(['Country_Region','Province_State'],axis=1 ).set_index('Date' ).drop('Fatalities',axis=1)
 cc.index.name=''
cc['EverydayCC']=cc.ConfirmedCases.diff()
cc.EverydayCC.iloc[0]=0.0
try:
model = VAR(endog=cc.drop('Id',axis=1))
model_fit = model.fit()
yhat = model_fit.forecast(model_fit.y, steps=33)
 final=pd.DataFrame(data=yhat,index=pd.date_range(start='29/3/2020', periods=33),columns=['ConfirmedCases','EverydayCC'])
 final.drop('ConfirmedCases',axis=1)
 final['ConfirmedCases']=final['EverydayCC'].apply(lambda x: cc.iloc[-1][1]+x)
 final.drop('EverydayCC',axis=1,inplace=True)
 final['Province_State']=i
w1=w1.append(final)
 w1['Country_Region']=cc1['Country_Region']
w1['ForecastId']=cc1['ForecastId']
p =p.append(w1)
 except:
print('Exception' )<filter>
cols_irr = [
'Property GFA - Self-Reported(ft²)',
'Weather Normalized Site EUI(kBtu/ft²)',
'Water Use(All Water Sources )(kgal)',
'Water Intensity(All Water Sources )(gal/ft²)',
'Electricity Use - Grid Purchase(kBtu)',
'Borough',
'Natural Gas Use(kBtu)',
'Fuel Oil
'Indirect GHG Emissions(Metric Tons CO2e)',
'Total GHG Emissions(Metric Tons CO2e)',
'Direct GHG Emissions(Metric Tons CO2e)',
'Occupancy',
'Weather Normalized Site Natural Gas Intensity(therms/ft²)',
'3rd Largest Property Use Type - Gross Floor Area(ft²)',
'Weather Normalized Site Natural Gas Use(therms)',
'Weather Normalized Site Electricity(kWh)',
'Number of Buildings - Self-reported',
'Fuel Oil
'Water Required?',
'Metered Areas(Energy)',
'Metered Areas(Water)',
'Fuel Oil
'Fuel Oil
'District Steam Use(kBtu)',
'Diesel
'3rd Largest Property Use Type'
]
dp3=p[1:] 
<drop_column>
X = X.drop(columns = cols_irr )
for i in pro:
t1=mdfw[['Province_State','Fatalities','Date']]
t1.index.name=''
t1=t1[t1['Province_State']==i]
w1=t1[t1['Date']>=datetime.strptime('2020-03-19', '%Y-%m-%d')]
w1=w1[t1['Date']<=datetime.strptime('2020-03-28', '%Y-%m-%d')].set_index('Date')
 w1.index.name=''
cc1=ndf[ndf['Province_State']==i].set_index('Date')
 cc=mdfw[mdfw['Province_State']==i].drop(['Country_Region','Province_State'],axis=1 ).set_index('Date' ).drop('ConfirmedCases',axis=1)
 cc.index.name=''
cc['EverydayCC']=cc.Fatalities.diff()
cc.EverydayCC.iloc[0]=0.0
try:
model = VAR(endog=cc.drop('Id',axis=1))
model_fit = model.fit()
yhat = model_fit.forecast(model_fit.y, steps=33)
 final=pd.DataFrame(data=yhat,index=pd.date_range(start='29/3/2020', periods=33),columns=['Fatalities','EverydayCC'])
 final.drop('Fatalities',axis=1)
 final['Fatalities']=final['EverydayCC'].apply(lambda x: cc.iloc[-1][1]+x)
 final.drop('EverydayCC',axis=1,inplace=True)
 final['Province_State']=i
w1=w1.append(final)
 w1['Country_Region']=cc1['Country_Region']
w1['ForecastId']=cc1['ForecastId']
p =p.append(w1)
 except:
print('Exception' )<define_variables>
scaler = MinMaxScaler()
cols = X.columns
X[cols] = scaler.fit_transform(X)
 X.head()
dp4=p[1:]<merge>
Xc = sm.add_constant(X)
 modelo_v1 = sm.OLS(y, Xc)
 modelo_v2 = modelo_v1.fit()
dp1=dp1.reset_index()
dp2=dp2.reset_index()
dp5=pd.merge(dp1,dp2,on=['Country_Region','Province_State','ForecastId','index'])
 dp3=dp3.reset_index()
dp4=dp4.reset_index()
dp6=pd.merge(dp3,dp4,on=['Country_Region','Province_State','ForecastId','index'])
 dp7=dp5.append(dp6)
 dp8=dp7.drop(['index','Country_Region','Province_State'],axis=1 ).sort_values('ForecastId' )<save_to_csv>
modelo = linear_model.LinearRegression(normalize = False, fit_intercept = True )
dp8['ForecastId']=dp8['ForecastId'].apply(lambda x: int(x))
dp8['a']=dp8['ConfirmedCases']
dp8['ConfirmedCases']=dp8['ForecastId']
dp8['ForecastId']=dp8['a']
dp8.drop('a',axis=1,inplace=True)
 dp8.columns=['ForecastId','ConfirmedCases','Fatalities']
dp8.to_csv('submission.csv',index=False )<load_from_csv>
def r2_est(X,y):
return r2_score(y, modelo.fit(X,y ).predict(X ).clip(1,100))
path = '/kaggle/input/covid19-global-forecasting-week-3'
train = pd.read_csv(os.path.join(path, 'train.csv'))
test = pd.read_csv(os.path.join(path, 'test.csv'))
subm = pd.read_csv(os.path.join(path, 'submission.csv'))<data_type_conversions>
def mean_absolute_error(y, y_pred):
return(1 / len(y)) * np.sum(np.abs(y - y_pred))
valid_date = pd.to_datetime('2020-04-10' )<data_type_conversions>
print('Baseline R2: %0.3f' % r2_est(X,y))
train['area'] = train['Country_Region'].astype(str)+ '_' + train['Province_State'].astype(str)
 test['area'] = test['Country_Region'].astype(str)+ '_' + test['Province_State'].astype(str)
 train['Date'] = pd.to_datetime(train['Date'] )<load_pretrained>
predictions = modelo.fit(X,y ).predict(X ).clip(1,100)
 predictions
path = '/kaggle/input/start-index-to-fit'
with open(os.path.join(path, 'dict_bst_ind.pickle'), 'rb')as f:
dict_bst_ind = pickle.load(f)
 with open(os.path.join(path, 'dict_bst_ind_Fat.pickle'), 'rb')as f:
dict_bst_ind_fat = pickle.load(f)
 with open(os.path.join(path, 'pop_dict.pickle'), 'rb')as f:
pop_dict = pickle.load(f )<string_transform>
def get_train_piece(area, valid_date):
data = train[(train.area == area)&(train.Date < valid_date)].reset_index()
data = data[data['ConfirmedCases'] > 0].reset_index(drop = True)
 return data<predict_on_test>
print('MAE test score: %0.3f' % mean_absolute_error(y, predictions))
pred_df = pd.DataFrame()
for pred_area in tqdm(test.area.unique()):
train_df = get_train_piece(pred_area, valid_date)
 len_train = train_df.shape[0]
population = -1
if pred_area in pop_dict:
population = pop_dict[pred_area]
test_df = test[test.area == pred_area].reset_index(drop = True)
 len_test = test_df.shape[0]
ans = pd.DataFrame()
ans['ForecastId'] = test_df['ForecastId'].values
if pred_area not in dict_bst_ind:
ans['ConfirmedCases'] = fit_predict(train_df['ConfirmedCases'].values, len_test, 0, population, 'cases')
 else:
ans['ConfirmedCases'] = fit_predict(train_df['ConfirmedCases'].values, len_test, dict_bst_ind[pred_area], population, 'cases')
 if pred_area not in dict_bst_ind_fat:
ans['Fatalities'] = fit_predict(train_df['Fatalities'].values, len_test, 0, population, 'fat')
 else:
ans['Fatalities'] = fit_predict(train_df['Fatalities'].values, len_test, dict_bst_ind_fat[pred_area], population, 'fat')
 pred_df = pd.concat([pred_df, ans], axis = 0 ).reset_index(drop = True )<save_to_csv>
num_folds = 10
num_instances = len(X)
 seed = 998
modelos = []
modelos.append(( 'LINEAR', LinearRegression()))
modelos.append(( 'RIDGE', Ridge()))
modelos.append(( 'LASSO', Lasso()))
modelos.append(( 'ELASTIC', ElasticNet()))
modelos.append(( 'KNN', KNeighborsRegressor()))
modelos.append(( 'CART', DecisionTreeRegressor()))
resultados = []
nomes = []
for nome, modelo in modelos:
kfold = model_selection.KFold(n_splits = num_folds, random_state = seed)
 cv_results = model_selection.cross_val_score(modelo, X, y, cv = kfold, scoring = 'neg_mean_absolute_error')
 resultados.append(cv_results)
 nomes.append(nome)
 msg = "%s: %.3f(%.3f)" %(nome, cv_results.mean() , cv_results.std())
 print(msg )
pred_df.to_csv('submission.csv', index=False )<set_options>
num_folds = 10
num_instances = len(X)
 seed = 998
modelos = []
modelos.append(( 'AD', AdaBoostRegressor()))
modelos.append(( 'BG', BaggingRegressor()))
modelos.append(( 'ET', ExtraTreesRegressor()))
modelos.append(( 'GB', GradientBoostingRegressor()))
modelos.append(( 'RF', RandomForestRegressor()))
modelos.append(( 'XGB', XGBRegressor()))
resultados = []
nomes = []
for nome, modelo in modelos:
kfold = model_selection.KFold(n_splits = num_folds, random_state = seed)
 cv_results = model_selection.cross_val_score(modelo, X, y, cv = kfold, scoring = 'neg_mean_absolute_error')
 resultados.append(cv_results)
 nomes.append(nome)
 msg = "%s: %.3f(%.3f)" %(nome, cv_results.mean() , cv_results.std())
 print(msg )
%matplotlib inline
InteractiveShell.ast_node_interactivity = "all"
pd.set_option('display.max_columns', 99)
 pd.set_option('display.max_rows', 99)
<set_options>
index = df_test['Property Id']
df_test = df_test.drop(columns = ['Property Id'] )
plt.rcParams['figure.figsize'] = [16, 10]
plt.rcParams['font.size'] = 14
sns.set_palette(sns.color_palette('tab20', 20)) 
<load_from_csv>
df_test = df_test.drop(columns = cols_irr )
COMP = '.. /input/covid19-global-forecasting-week-3'
DATEFORMAT = '%Y-%m-%d'
def get_comp_data(COMP):
train = pd.read_csv(f'{COMP}/train.csv')
 test = pd.read_csv(f'{COMP}/test.csv')
 submission = pd.read_csv(f'{COMP}/submission.csv')
 print(train.shape, test.shape, submission.shape)
 train['Country_Region'] = train['Country_Region'].str.replace(',', '')
 test['Country_Region'] = test['Country_Region'].str.replace(',', '')
 train['Location'] = train['Country_Region'] + '-' + train['Province_State'].fillna('')
 test['Location'] = test['Country_Region'] + '-' + test['Province_State'].fillna('')
 train['LogConfirmed'] = to_log(train.ConfirmedCases)
 train['LogFatalities'] = to_log(train.Fatalities)
 train = train.drop(columns=['Province_State'])
 test = test.drop(columns=['Province_State'])
 country_codes = pd.read_csv('.. /input/covid19-metadata/country_codes.csv', keep_default_na=False)
 train = train.merge(country_codes, on='Country_Region', how='left')
 test = test.merge(country_codes, on='Country_Region', how='left')
 train['DateTime'] = pd.to_datetime(train['Date'])
 test['DateTime'] = pd.to_datetime(test['Date'])
 return train, test, submission
def process_each_location(df):
dfs = []
for loc, df in tqdm(df.groupby('Location')) :
df = df.sort_values(by='Date')
 df['Fatalities'] = df['Fatalities'].cummax()
df['ConfirmedCases'] = df['ConfirmedCases'].cummax()
df['LogFatalities'] = df['LogFatalities'].cummax()
df['LogConfirmed'] = df['LogConfirmed'].cummax()
df['LogConfirmedNextDay'] = df['LogConfirmed'].shift(-1)
 df['ConfirmedNextDay'] = df['ConfirmedCases'].shift(-1)
 df['DateNextDay'] = df['Date'].shift(-1)
 df['LogFatalitiesNextDay'] = df['LogFatalities'].shift(-1)
 df['FatalitiesNextDay'] = df['Fatalities'].shift(-1)
 df['LogConfirmedDelta'] = df['LogConfirmedNextDay'] - df['LogConfirmed']
df['ConfirmedDelta'] = df['ConfirmedNextDay'] - df['ConfirmedCases']
df['LogFatalitiesDelta'] = df['LogFatalitiesNextDay'] - df['LogFatalities']
df['FatalitiesDelta'] = df['FatalitiesNextDay'] - df['Fatalities']
dfs.append(df)
 return pd.concat(dfs)
 def add_days(d, k):
return dt.datetime.strptime(d, DATEFORMAT)+ dt.timedelta(days=k)
 def to_log(x):
return np.log(x + 1)
 def to_exp(x):
return np.exp(x)- 1 
<count_duplicates>
X_test = df_test
train[train.geo_region.isna() ].Country_Region.unique()
train = train.fillna('
test = test.fillna('
train[train.duplicated(['Date', 'Location'])]
train.count()<save_to_csv>
scaler = MinMaxScaler()
cols = X_test.columns
X_test[cols] = scaler.fit_transform(X_test)
 X_test.head()
train = train.sort_values(by='Date')
 countries_latest_state = train[train['Date'] == TRAIN_END].groupby([
'Country_Region', 'continent', 'geo_region', 'country_iso_code_3'] ).sum() [[
'ConfirmedCases', 'Fatalities']].reset_index()
countries_latest_state['Log10Confirmed'] = np.log10(countries_latest_state.ConfirmedCases + 1)
 countries_latest_state['Log10Fatalities'] = np.log10(countries_latest_state.Fatalities + 1)
 countries_latest_state = countries_latest_state.sort_values(by='Fatalities', ascending=False)
 countries_latest_state.to_csv('countries_latest_state.csv', index=False)
 countries_latest_state.shape
countries_latest_state.head()<merge>
def r2_est(X,y):
return r2_score(y, modelo.fit(X,y ).predict(X ).clip(1,100))
latest_loc = train[train['Date'] == TRAIN_END][['Location', 'ConfirmedCases', 'Fatalities']]
max_loc = train.groupby(['Location'])[['ConfirmedCases', 'Fatalities']].max().reset_index()
check = pd.merge(latest_loc, max_loc, on='Location')
 np.mean(check.ConfirmedCases_x == check.ConfirmedCases_y)
 np.mean(check.Fatalities_x == check.Fatalities_y)
 check[check.Fatalities_x != check.Fatalities_y]
check[check.ConfirmedCases_x != check.ConfirmedCases_y]<feature_engineering>
def mean_absolute_error(y, y_pred):
return(1 / len(y)) * np.sum(np.abs(y - y_pred))
regional_progress = train_clean.groupby(['DateTime', 'continent'] ).sum() [['ConfirmedCases', 'Fatalities']].reset_index()
regional_progress['Log10Confirmed'] = np.log10(regional_progress.ConfirmedCases + 1)
 regional_progress['Log10Fatalities'] = np.log10(regional_progress.Fatalities + 1)
 regional_progress = regional_progress[regional_progress.continent != '<merge>
modelo = GradientBoostingRegressor()
countries_0301 = country_progress[country_progress.Date == '2020-03-01'][[
'Country_Region', 'ConfirmedCases', 'Fatalities']]
countries_0331 = country_progress[country_progress.Date == '2020-03-31'][[
'Country_Region', 'ConfirmedCases', 'Fatalities']]
countries_in_march = pd.merge(countries_0301, countries_0331, on='Country_Region', suffixes=['_0301', '_0331'])
 countries_in_march['IncreaseInMarch'] = countries_in_march.ConfirmedCases_0331 /(countries_in_march.ConfirmedCases_0301 + 1)
 countries_in_march = countries_in_march[countries_in_march.ConfirmedCases_0331 > 200].sort_values(
 by='IncreaseInMarch', ascending=False)
 countries_in_march.tail(15 )<save_to_csv>
model_fit = modelo.fit(X,y )
train_clean['Geo
latest = train_clean[train_clean.Date == TRAIN_END][[
'Geo
daily_confirmed_deltas = train_clean[train_clean.Date >= '2020-03-17'].pivot(
 'Geo
daily_confirmed_deltas = latest.merge(daily_confirmed_deltas, on='Geo
daily_confirmed_deltas.shape
daily_confirmed_deltas.head()
daily_confirmed_deltas.to_csv('daily_confirmed_deltas.csv', index=False )<save_to_csv>
predictions = model_fit.predict(X ).clip(1,100)
 predictions
deltas = train_clean[np.logical_and(
 train_clean.LogConfirmed > 2,
~train_clean.Location.str.startswith('China')
)].dropna().sort_values(by='LogConfirmedDelta', ascending=False)
 deltas['start'] = deltas['LogConfirmed'].round(0)
 confirmed_deltas = pd.concat([
deltas.groupby('start')[['LogConfirmedDelta']].mean() ,
deltas.groupby('start')[['LogConfirmedDelta']].std() ,
deltas.groupby('start')[['LogConfirmedDelta']].count()
], axis=1)
 deltas.mean()
confirmed_deltas.columns = ['avg', 'std', 'cnt']
confirmed_deltas
confirmed_deltas.to_csv('confirmed_deltas.csv' )<feature_engineering>
print('Baseline R2: %0.3f' % r2_est(X,y))
DECAY = 0.93
DECAY ** 7, DECAY ** 14, DECAY ** 21, DECAY ** 28
confirmed_deltas = train.groupby(['Location', 'Country_Region', 'continent'])[[
'Id']].count().reset_index()
GLOBAL_DELTA = 0.11
confirmed_deltas['DELTA'] = GLOBAL_DELTA
confirmed_deltas.loc[confirmed_deltas.continent=='Africa', 'DELTA'] = 0.14
confirmed_deltas.loc[confirmed_deltas.continent=='Oceania', 'DELTA'] = 0.06
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Korea South', 'DELTA'] = 0.011
confirmed_deltas.loc[confirmed_deltas.Country_Region=='US', 'DELTA'] = 0.15
confirmed_deltas.loc[confirmed_deltas.Country_Region=='China', 'DELTA'] = 0.01
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Japan', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Singapore', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Taiwan*', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Switzerland', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Norway', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Iceland', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Austria', 'DELTA'] = 0.06
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Italy', 'DELTA'] = 0.04
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Spain', 'DELTA'] = 0.08
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Portugal', 'DELTA'] = 0.12
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Israel', 'DELTA'] = 0.12
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Iran', 'DELTA'] = 0.08
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Germany', 'DELTA'] = 0.07
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Malaysia', 'DELTA'] = 0.06
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Russia', 'DELTA'] = 0.18
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Ukraine', 'DELTA'] = 0.18
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Brazil', 'DELTA'] = 0.12
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Turkey', 'DELTA'] = 0.18
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Philippines', 'DELTA'] = 0.18
confirmed_deltas.loc[confirmed_deltas.Location=='France-', 'DELTA'] = 0.1
confirmed_deltas.loc[confirmed_deltas.Location=='United Kingdom-', 'DELTA'] = 0.12
confirmed_deltas.loc[confirmed_deltas.Location=='Diamond Princess-', 'DELTA'] = 0.00
confirmed_deltas.loc[confirmed_deltas.Location=='China-Hong Kong', 'DELTA'] = 0.08
confirmed_deltas.loc[confirmed_deltas.Location=='San Marino-', 'DELTA'] = 0.03
confirmed_deltas.shape, confirmed_deltas.DELTA.mean()
confirmed_deltas[confirmed_deltas.DELTA != GLOBAL_DELTA].shape, confirmed_deltas[confirmed_deltas.DELTA != GLOBAL_DELTA].DELTA.mean()
confirmed_deltas[confirmed_deltas.DELTA != GLOBAL_DELTA]
confirmed_deltas.describe()<save_to_csv>
print('MAE test score: %0.3f' % mean_absolute_error(y, predictions))
daily_log_confirmed = train_clean.pivot('Location', 'Date', 'LogConfirmed' ).reset_index()
daily_log_confirmed = daily_log_confirmed.sort_values(TRAIN_END, ascending=False)
 daily_log_confirmed.to_csv('daily_log_confirmed.csv', index=False)
 for i, d in tqdm(enumerate(pd.date_range(add_days(TRAIN_END, 1), add_days(TEST_END, 1)))) :
new_day = str(d ).split(' ')[0]
last_day = dt.datetime.strptime(new_day, DATEFORMAT)- dt.timedelta(days=1)
 last_day = last_day.strftime(DATEFORMAT)
 for loc in confirmed_deltas.Location.values:
confirmed_delta = confirmed_deltas.loc[confirmed_deltas.Location == loc, 'DELTA'].values[0]
daily_log_confirmed.loc[daily_log_confirmed.Location == loc, new_day] = daily_log_confirmed.loc[daily_log_confirmed.Location == loc, last_day] + \
confirmed_delta * DECAY ** i<save_to_csv>
predictions = model_fit.predict(X_test ).clip(1,100)
 predictions
<feature_engineering><EOS> 
submission = pd.DataFrame()
submission['Property Id'] = index
submission['score'] = predictions.round().astype(int)
 submission.to_csv('submission.csv', index = False )