Kaggle competition: Bike Sharing Demand
submission = pd.DataFrame({"row_id": row_ids, "meter_reading": np.clip(pred, 0, a_max=None)})
 submission.to_csv("submission.csv", index=False )<set_options>
%matplotlib inline
warnings.filterwarnings("ignore", category=FutureWarning)
 warnings.filterwarnings("ignore" )
py.init_notebook_mode(connected=True)
<load_from_csv>
train = pd.read_csv("/kaggle/input/bike-sharing-demand/train.csv")
 test = pd.read_csv("/kaggle/input/bike-sharing-demand/test.csv" )
RAW_DATA_DIR = '/kaggle/input/ashrae-energy-prediction/'
weather_dtypes = {
'site_id': np.uint8,
'air_temperature': np.float32,
'cloud_coverage': np.float32,
'dew_temperature': np.float32,
'precip_depth_1_hr': np.float32,
'sea_level_pressure': np.float32,
'wind_direction': np.float32,
'wind_speed': np.float32,
}
weather_train = pd.read_csv(pjoin(RAW_DATA_DIR, 'weather_train.csv'),dtype=weather_dtypes,
parse_dates=['timestamp'])
 weather_test = pd.read_csv(pjoin(RAW_DATA_DIR, 'weather_test.csv'),dtype=weather_dtypes,
parse_dates=['timestamp'])
 weather = pd.concat([weather_train,weather_test],ignore_index=True)
 del weather_train, weather_test
weather_key = ['site_id', 'timestamp']
temp_skeleton = weather[weather_key + ['air_temperature']].drop_duplicates(subset=weather_key ).sort_values(by=weather_key ).copy()
del weather<feature_engineering>
train.isnull().sum()
temp_skeleton['temp_rank'] = temp_skeleton.groupby(['site_id', temp_skeleton.timestamp.dt.date])['air_temperature'].rank('average')
 df_2d = temp_skeleton.groupby(['site_id', temp_skeleton.timestamp.dt.hour])['temp_rank'].mean().unstack(level=1)
 site_ids_offsets = pd.Series(df_2d.values.argmax(axis=1)- 14)
 site_ids_offsets.index.name = 'site_id'
def timestamp_align(df):
df['offset'] = df.site_id.map(site_ids_offsets)
 df['timestamp_aligned'] =(df.timestamp - pd.to_timedelta(df.offset, unit='H'))
df['timestamp'] = df['timestamp_aligned']
del df['timestamp_aligned']
return df<set_options>
train.isnull().sum()
def reduce_mem_usage(df, use_float16=False):

 start_mem = df.memory_usage().sum() / 1024**2
print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))
for col in df.columns:
if is_datetime(df[col])or is_categorical_dtype(df[col]):
continue
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if use_float16 and c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype('category')
 end_mem = df.memory_usage().sum() / 1024**2
print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))
print('Decreased by {:.1f}%'.format(100 *(start_mem - end_mem)/ start_mem))
return df<load_from_disk>
target = train['count']
%%time
root = Path('.. /input/ashrae-feather-format-for-fast-loading')
 train_df = pd.read_feather(root/'train.feather')
 weather_train_df = pd.read_feather(root/'weather_train.feather')
 building_meta_df = pd.read_feather(root/'building_metadata.feather' )<feature_engineering>
from scipy import stats
from scipy.stats import norm
building_site_dict = dict(zip(building_meta_df['building_id'], building_meta_df['site_id']))
site_meter_raw = train_df[['building_id', 'meter', 'timestamp', 'meter_reading']].copy()
site_meter_raw['site_id'] = site_meter_raw.building_id.map(building_site_dict)
 del site_meter_raw['building_id']
site_meter_to_plot = site_meter_raw.copy()
site_meter_to_plot["hour"] = site_meter_to_plot["timestamp"].dt.hour
elec_to_plot = site_meter_to_plot[site_meter_to_plot.meter == 0]<feature_engineering>
train["log_count"] = np.log(target+1 )
def preprocess(df):
df["hour"] = df["timestamp"].dt.hour
df["weekend"] = df["timestamp"].dt.weekday
df["month"] = df["timestamp"].dt.month
df["dayofweek"] = df["timestamp"].dt.dayofweek
def add_lag_feature(weather_df, window=3):
group_df = weather_df.groupby('site_id')
 cols = ['air_temperature', 'cloud_coverage', 'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure', 'wind_direction', 'wind_speed']
rolled = group_df[cols].rolling(window=window, min_periods=0)
 lag_mean = rolled.mean().reset_index().astype(np.float16)
 lag_max = rolled.max().reset_index().astype(np.float16)
 lag_min = rolled.min().reset_index().astype(np.float16)
 lag_std = rolled.std().reset_index().astype(np.float16)
 for col in cols:
weather_df[f'{col}_mean_lag{window}'] = lag_mean[col]
weather_df[f'{col}_max_lag{window}'] = lag_max[col]
weather_df[f'{col}_min_lag{window}'] = lag_min[col]
weather_df[f'{col}_std_lag{window}'] = lag_std[col]
train_df['date'] = train_df['timestamp'].dt.date
train_df['meter_reading_log1p'] = np.log1p(train_df['meter_reading'])
 building_meta_df[building_meta_df.site_id == 0]
train_df = train_df.query('not(building_id <= 104 & meter == 0 & timestamp <= "2016-05-20")')
 debug = False
preprocess(train_df)
 df_group = train_df.groupby('building_id')['meter_reading_log1p']
building_mean = df_group.mean().astype(np.float16)
 building_median = df_group.median().astype(np.float16)
 building_min = df_group.min().astype(np.float16)
 building_max = df_group.max().astype(np.float16)
 building_std = df_group.std().astype(np.float16)
 train_df['building_mean'] = train_df['building_id'].map(building_mean)
 train_df['building_median'] = train_df['building_id'].map(building_median)
 train_df['building_min'] = train_df['building_id'].map(building_min)
 train_df['building_max'] = train_df['building_id'].map(building_max)
 train_df['building_std'] = train_df['building_id'].map(building_std)
 weather_train_df = timestamp_align(weather_train_df)
 weather_train_df = weather_train_df.groupby('site_id' ).apply(lambda group: group.interpolate(limit_direction='both'))
add_lag_feature(weather_train_df, window=3)
 add_lag_feature(weather_train_df, window=72)
 primary_use_list = building_meta_df['primary_use'].unique()
primary_use_dict = {key: value for value, key in enumerate(primary_use_list)}
print('primary_use_dict: ', primary_use_dict)
 building_meta_df['primary_use'] = building_meta_df['primary_use'].map(primary_use_dict)
 gc.collect()
reduce_mem_usage(train_df, use_float16=True)
 reduce_mem_usage(building_meta_df, use_float16=True)
 reduce_mem_usage(weather_train_df, use_float16=True )<define_variables>
feature_names=list(test)
 df_train=train[feature_names]
df=pd.concat(( df_train, test))
category_cols = ['building_id', 'site_id', 'primary_use']
feature_cols = ['square_feet', 'year_built'] + [
'hour', 'weekend',
'building_median'] + [
'air_temperature', 'cloud_coverage',
'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure',
'wind_direction', 'wind_speed', 'air_temperature_mean_lag72',
'air_temperature_max_lag72', 'air_temperature_min_lag72',
'air_temperature_std_lag72', 'cloud_coverage_mean_lag72',
'dew_temperature_mean_lag72', 'precip_depth_1_hr_mean_lag72',
'sea_level_pressure_mean_lag72', 'wind_direction_mean_lag72',
'wind_speed_mean_lag72', 'air_temperature_mean_lag3',
'air_temperature_max_lag3',
'air_temperature_min_lag3', 'cloud_coverage_mean_lag3',
'dew_temperature_mean_lag3',
'precip_depth_1_hr_mean_lag3', 'sea_level_pressure_mean_lag3',
'wind_direction_mean_lag3', 'wind_speed_mean_lag3']<train_model>
import datetime
def create_X_y(train_df, target_meter):
target_train_df = train_df[train_df['meter'] == target_meter]
target_train_df = target_train_df.merge(building_meta_df, on='building_id', how='left')
 target_train_df = target_train_df.merge(weather_train_df, on=['site_id', 'timestamp'], how='left')
 X_train = target_train_df[feature_cols + category_cols]
y_train = target_train_df['meter_reading_log1p'].values
del target_train_df
return X_train, y_train
def fit_lgbm(train, val, devices=(-1,), seed=None, cat_features=None, num_rounds=1500, lr=0.1, bf=0.1):

 X_train, y_train = train
X_valid, y_valid = val
metric = 'l2'
params = {'num_leaves': 31,
'objective': 'regression',
'learning_rate': lr,
"boosting": "gbdt",
"bagging_freq": 5,
"bagging_fraction": bf,
"feature_fraction": 0.9,
"metric": metric,
}
device = devices[0]
if device == -1:
pass
else:
print(f'using gpu device_id {device}...')
 params.update({'device': 'gpu', 'gpu_device_id': device})
 params['seed'] = seed
early_stop = 20
verbose_eval = 20
d_train = lgb.Dataset(X_train, label=y_train, categorical_feature=cat_features)
 d_valid = lgb.Dataset(X_valid, label=y_valid, categorical_feature=cat_features)
 watchlist = [d_train, d_valid]
print('training LGB:')
 model = lgb.train(params,
train_set=d_train,
num_boost_round=num_rounds,
valid_sets=watchlist,
verbose_eval=verbose_eval,
early_stopping_rounds=early_stop)
 y_pred_valid = model.predict(X_valid, num_iteration=model.best_iteration)
 print('best_score', model.best_score)
 log = {'train/mae': model.best_score['training']['l2'],
'valid/mae': model.best_score['valid_1']['l2']}
return model, y_pred_valid, log
folds = 5
seed = 666
shuffle = False
kf = KFold(n_splits=folds, shuffle=shuffle, random_state=seed )<split>
tmp = pd.to_datetime(train['datetime'] )
target_meter = 0
X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models0 = []
for train_idx, valid_idx in kf.split(X_train, y_train):
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols,
num_rounds=1000, lr=0.05, bf=0.7)
 y_valid_pred_total[valid_idx] = y_pred_valid
models0.append(model)
 gc.collect()
if debug:
break
sns.distplot(y_train)
 del X_train, y_train
gc.collect()<split>
df['datetime'] = pd.to_datetime(df['datetime'])
 df['day'] = df['datetime'].dt.day
df['hour'] = df['datetime'].dt.hour
df['dayofweek'] = df['datetime'].dt.dayofweek
df['month'] = df['datetime'].dt.month
df['year'] = df['datetime'].dt.year
df['weekend'] =(df['dayofweek'] ==5)|(df['dayofweek'] == 6 )
target_meter = 1
X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models1 = []
for train_idx, valid_idx in kf.split(X_train, y_train):
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols, num_rounds=1000,
lr=0.05, bf=0.5)
 y_valid_pred_total[valid_idx] = y_pred_valid
models1.append(model)
 gc.collect()
if debug:
break
sns.distplot(y_train)
 del X_train, y_train
gc.collect()<split>
train['datetime'] = pd.to_datetime(train['datetime'])
 train['day'] = train['datetime'].dt.day
train['hour'] = train['datetime'].dt.hour
train['dayofweek'] = train['datetime'].dt.dayofweek
train['month'] = train['datetime'].dt.month
train['year'] = train['datetime'].dt.year
train['weekend'] =(train['dayofweek'] ==5)|(train['dayofweek'] == 6 )
target_meter = 2
X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models2 = []
for train_idx, valid_idx in kf.split(X_train, y_train):
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols,
num_rounds=1000, lr=0.05, bf=0.8)
 y_valid_pred_total[valid_idx] = y_pred_valid
models2.append(model)
 gc.collect()
if debug:
break
sns.distplot(y_train)
 del X_train, y_train
gc.collect()<split>
df.drop(['datetime'], axis=1, inplace=True )
target_meter = 3
X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models3 = []
for train_idx, valid_idx in kf.split(X_train, y_train):
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols, num_rounds=1000,
lr=0.03, bf=0.9)
 y_valid_pred_total[valid_idx] = y_pred_valid
models3.append(model)
 gc.collect()
if debug:
break
sns.distplot(y_train)
 del X_train, y_train
gc.collect()<categorify>
df=df.drop(columns=['month', 'day'] )
print('loading...')
 test_df = pd.read_feather(root/'test.feather')
 weather_test_df = pd.read_feather(root/'weather_test.feather')
 print('preprocessing building...')
 test_df['date'] = test_df['timestamp'].dt.date
preprocess(test_df)
 test_df['building_mean'] = test_df['building_id'].map(building_mean)
 test_df['building_median'] = test_df['building_id'].map(building_median)
 test_df['building_min'] = test_df['building_id'].map(building_min)
 test_df['building_max'] = test_df['building_id'].map(building_max)
 test_df['building_std'] = test_df['building_id'].map(building_std)
 print('preprocessing weather...')
 weather_test_df = timestamp_align(weather_test_df)
 weather_test_df = weather_test_df.groupby('site_id' ).apply(lambda group: group.interpolate(limit_direction='both'))
weather_test_df.groupby('site_id' ).apply(lambda group: group.isna().sum())
 add_lag_feature(weather_test_df, window=3)
 add_lag_feature(weather_test_df, window=72)
 print('reduce mem usage...')
 reduce_mem_usage(test_df, use_float16=True)
 reduce_mem_usage(weather_test_df, use_float16=True)
 gc.collect()<load_from_disk>
season_encoded = pd.get_dummies(df['season'],prefix= 'season')
 df = pd.concat(( df,season_encoded), axis=1)
 df = df.drop(columns = 'season' )
sample_submission = pd.read_feather(os.path.join(root, 'sample_submission.feather'))
reduce_mem_usage(sample_submission )<merge>
df['holiday'] = df['holiday']
def create_X(test_df, target_meter):
target_test_df = test_df[test_df['meter'] == target_meter]
target_test_df = target_test_df.merge(building_meta_df, on='building_id', how='left')
 target_test_df = target_test_df.merge(weather_test_df, on=['site_id', 'timestamp'], how='left')
 X_test = target_test_df[feature_cols + category_cols]
return X_test<predict_on_test>
df['workingday'] = df['workingday']
def pred(X_test, models, batch_size=1000000):
iterations =(X_test.shape[0] + batch_size -1)// batch_size
print('iterations', iterations)
 y_test_pred_total = np.zeros(X_test.shape[0])
 for i, model in enumerate(models):
print(f'predicting {i}-th model')
 for k in tqdm(range(iterations)) :
y_pred_test = model.predict(X_test[k*batch_size:(k+1)*batch_size], num_iteration=model.best_iteration)
 y_test_pred_total[k*batch_size:(k+1)*batch_size] += y_pred_test
y_test_pred_total /= len(models)
 return y_test_pred_total<feature_engineering>
weather_encoded = pd.get_dummies(df['weather'],prefix= 'weather')
 df = pd.concat(( df,weather_encoded), axis=1)
 df = df.drop(columns = 'weather' )
sample_submission.loc[test_df['meter'] == 0, 'meter_reading'] = np.expm1(y_test0)
 sample_submission.loc[test_df['meter'] == 1, 'meter_reading'] = np.expm1(y_test1)
 sample_submission.loc[test_df['meter'] == 2, 'meter_reading'] = np.expm1(y_test2)
 sample_submission.loc[test_df['meter'] == 3, 'meter_reading'] = np.expm1(y_test3 )<save_to_csv>
skew = df.apply(lambda x: skew(x))
skew.sort_values(ascending = False )
sample_submission.to_csv('submission.csv', index=False, float_format='%.4f')
 sample_submission.head()<import_modules>
skew = skew[abs(skew)> 0.5]
skew
PIXELS = 16
imageSize = PIXELS * PIXELS
num_features = imageSize
def load_train_cv(encoder):
X_train = []
y_train = []
print('Read train images')
 for j in range(10):
print('Load folder c{}'.format(j))
path = os.path.join('.. ', 'input', 'train', 'c' + str(j), '*.jpg')
 files = glob.glob(path)
 for fl in files:
img = cv2.imread(fl,0)
 img = cv2.resize(img,(PIXELS, PIXELS))
img = np.reshape(img,(1, num_features))
X_train.append(img)
 y_train.append(j)
 X_train = np.array(X_train)
 y_train = np.array(y_train)
 y_train = encoder.fit_transform(y_train ).astype('int32')
 X_train, y_train = shuffle(X_train, y_train)
 X_train, X_test, y_train, y_test = train_test_split(X_train, y_train, test_size=0.1)
 X_train = X_train.reshape(X_train.shape[0], 1, PIXELS, PIXELS ).astype('float32')/ 255.
 X_test = X_test.reshape(X_test.shape[0], 1, PIXELS, PIXELS ).astype('float32')/ 255.
 return X_train, y_train, X_test, y_test, encoder
def load_test() :
print('Read test images')
 path = os.path.join('.. ', 'input', 'test', '*.jpg')
 files = glob.glob(path)
 X_test = []
X_test_id = []
total = 0
thr = math.floor(len(files)/10)
 for fl in files:
flbase = os.path.basename(fl)
 img = cv2.imread(fl,0)
 img = cv2.resize(img,(PIXELS, PIXELS))
img = np.reshape(img,(1, num_features))
X_test.append(img)
 X_test_id.append(flbase)
 total += 1
if total%thr == 0:
print('Read {} images from {}'.format(total, len(files)))
 X_test = np.array(X_test)
 X_test_id = np.array(X_test_id)
 X_test = X_test.reshape(X_test.shape[0], 1, PIXELS, PIXELS ).astype('float32')/ 255.
 return X_test, X_test_id

 def ZFTurboNet(input_var=None):
l_in = InputLayer(shape=(None, 1, PIXELS, PIXELS), input_var=input_var)
 l_conv = ConvLayer(l_in, num_filters=8, filter_size=3, pad=1, nonlinearity=rectify)
 l_convb = ConvLayer(l_conv, num_filters=8, filter_size=3, pad=1, nonlinearity=rectify)
 l_pool = MaxPool2DLayer(l_convb, pool_size=2)
 l_hidden = DenseLayer(l_pool, num_units=128, nonlinearity=rectify)
 l_out = DenseLayer(l_hidden, num_units=10, nonlinearity=softmax)
 return l_out
def iterate_minibatches(inputs, targets, batchsize):
assert len(inputs)== len(targets)
 indices = np.arange(len(inputs))
np.random.shuffle(indices)
 for start_idx in range(0, len(inputs)- batchsize + 1, batchsize):
excerpt = indices[start_idx:start_idx + batchsize]
yield inputs[excerpt], targets[excerpt]<compute_train_metric>
from statsmodels.stats.outliers_influence import variance_inflation_factor
BATCHSIZE = 32
LR = 0.001
ITERS = 2
X = T.tensor4('X')
 Y = T.ivector('y')
 output_layer = ZFTurboNet(X)
 output_train = lasagne.layers.get_output(output_layer)
 output_test = lasagne.layers.get_output(output_layer, deterministic=True)
 loss = lasagne.objectives.categorical_crossentropy(output_train, Y)
 loss = loss.mean()
valid_loss = lasagne.objectives.categorical_crossentropy(output_test, Y)
 valid_loss = valid_loss.mean()
valid_acc = T.mean(T.eq(T.argmax(output_test, axis=1), Y), dtype=theano.config.floatX)
 params = lasagne.layers.get_all_params(output_layer, trainable=True)
 updates = adam(loss, params, learning_rate=LR)
 train_fn = theano.function(inputs=[X,Y], outputs=loss, updates=updates)
 valid_fn = theano.function(inputs=[X,Y], outputs=[valid_loss, valid_acc])
 predict_proba = theano.function(inputs=[X], outputs=output_test)
<categorify>
vif = pd.DataFrame()
vif['Features'] = vif_data.columns
vif['vif'] = [variance_inflation_factor(
 vif_data.values, i)for i in range(vif_data.shape[1])]
vif.sort_values(by='vif',ascending=False )
encoder = LabelEncoder()
train_X, train_y, valid_X, valid_y, encoder = load_train_cv(encoder)
 print('Train shape:', train_X.shape, 'Test shape:', valid_X.shape)
 X_test, X_test_id = load_test() 
<train_model>
pca=PCA(n_components=1)
 pca.fit(df[['temp', 'atemp']] )
try:
for epoch in range(ITERS):
start = time.time()
train_loss = []
for batch in iterate_minibatches(train_X, train_y, BATCHSIZE):
inputs, targets = batch
train_loss.append(train_fn(inputs, targets))
train_loss = np.mean(train_loss)
 valid_loss = []
valid_acc = []
for batch in iterate_minibatches(valid_X, valid_y, BATCHSIZE):
inputs, targets = batch
valid_eval = valid_fn(inputs, targets)
 valid_loss.append(valid_eval[0])
 valid_acc.append(valid_eval[1])
 valid_loss = np.mean(valid_loss)
 valid_acc = np.mean(valid_acc)
 ratio = train_loss / valid_loss
end = time.time() - start
print('iter:', epoch, '| TL:', np.round(train_loss,decimals=3), '| VL:', np.round(valid_loss, decimals=3), '| Vacc:', np.round(valid_acc, decimals=3), '| Ratio:', np.round(ratio, decimals=2), '| Time:', np.round(end, decimals=1))
except KeyboardInterrupt:
pass 
<define_variables>
pca.explained_variance_ratio_
print('Making predictions')
 PRED_BATCH = 2
def iterate_pred_minibatches(inputs, batchsize):
for start_idx in range(0, len(inputs)- batchsize + 1, batchsize):
excerpt = slice(start_idx, start_idx + batchsize)
 yield inputs[excerpt]
predictions = []
for pred_batch in iterate_pred_minibatches(X_test, PRED_BATCH):
predictions.extend(predict_proba(pred_batch))
predictions = np.array(predictions)
 print('pred shape')
 print(predictions.shape)
 print('Creating Submission')
 def create_submission(predictions, test_id):
result1 = pd.DataFrame(predictions, columns=['c0', 'c1', 'c2', 'c3', 'c4', 'c5', 'c6', 'c7', 'c8', 'c9'])
 result1.loc[:, 'img'] = pd.Series(test_id, index=result1.index)
 result1.to_csv('submission_ZFTurboNet.csv', index=False)
 create_submission(predictions, X_test_id )<set_options>
df['pca']=pca.fit_transform(df[['temp','atemp']] )
%matplotlib inline
warnings.filterwarnings("ignore" )<load_from_disk>
stats.pearsonr(train['temp'],target )
df_train0 = pd.read_json('.. /input/deepfake-detection-challenge/metadata0.json' )<define_variables>
df.loc[df['windspeed']==0, 'windspeed']=df['windspeed'].mean()
test_dir = "/kaggle/input/deepfake-detection-challenge/test_videos/"
test_videos = sorted([x for x in os.listdir(test_dir)if x[-4:] == ".mp4"])
 len(test_videos )<import_modules>
df = df.drop(columns=['temp','atemp'] )
print("PyTorch version:", torch.__version__)
 print("CUDA version:", torch.version.cuda)
 print("cuDNN version:", torch.backends.cudnn.version() )<set_options>
from sklearn.model_selection import KFold, cross_val_score, train_test_split, GridSearchCV
from sklearn.metrics import mean_squared_log_error
from sklearn.preprocessing import RobustScaler
from sklearn.pipeline import make_pipeline
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from xgboost import XGBRegressor
from lightgbm import LGBMRegressor
from sklearn.linear_model import ElasticNet, Lasso, LinearRegression
gpu = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 gpu<load_pretrained>
new_train = df[:train.shape[0]]
new_test = df[train.shape[0]:]
facedet = BlazeFace().to(gpu)
 facedet.load_weights("/kaggle/input/blazeface-pytorch/blazeface.pth")
 facedet.load_anchors("/kaggle/input/blazeface-pytorch/anchors.npy")
 _ = facedet.train(False )<load_pretrained>
target = train['log_count']
frames_per_video = 20
video_reader = VideoReader()
video_read_fn = lambda x: video_reader.read_frames(x, num_frames=frames_per_video)
 face_extractor = FaceExtractor(video_read_fn, facedet)
 mean = [0.485, 0.456, 0.406]
std = [0.229, 0.224, 0.225]
normalize_transform = Normalize(mean, std )<choose_model_class>
X_train, X_val, y_train, y_val = train_test_split(new_train, target, test_size=0.2, shuffle=True )
class MyResNeXt(models.resnet.ResNet):
def __init__(self, training=True):
super(MyResNeXt, self ).__init__(block=models.resnet.Bottleneck,
layers=[3, 4, 6, 3],
groups=32,
width_per_group=4)
 self.fc = nn.Linear(2048, 1 )<set_options>
def rmsle_score(preds, true):
rmsle_score =(np.sum(( np.log1p(preds)-np.log1p(true)) **2)/len(true)) **0.5
return rmsle_score
<predict_on_test>
RMSLE = make_scorer(rmsle_score )
def predict_on_video(video_path, batch_size):
try:
faces = face_extractor.process_video(video_path)
 face_extractor.keep_only_best_face(faces)
 if len(faces)> 0:
x = np.zeros(( batch_size, input_size, input_size, 3), dtype=np.uint8)
 n = 0
for frame_data in faces:
for face in frame_data["faces"]:
resized_face = isotropically_resize_image(face, input_size)
 resized_face = make_square_image(resized_face)
 if n < batch_size:
x[n] = resized_face
n += 1
else:
print("WARNING: have %d faces but batch size is %d" %(n, batch_size))
if n > 0:
x = torch.tensor(x, device=gpu ).float()
x = x.permute(( 0, 3, 1, 2))
for i in range(len(x)) :
x[i] = normalize_transform(x[i] / 255.)
with torch.no_grad() :
y_pred = model(x)
 y_pred = torch.sigmoid(y_pred.squeeze())
 return y_pred[:n].mean().item()
except Exception as e:
print("Prediction error on video %s: %s" %(video_path, str(e)))
 return 0.5<predict_on_test>
import statsmodels.api as sm
def predict_on_video_set(videos, num_workers):
def process_file(i):
filename = videos[i]
y_pred = predict_on_video(os.path.join(test_dir, filename), batch_size=frames_per_video)
 return y_pred
with ThreadPoolExecutor(max_workers=num_workers)as ex:
predictions = ex.map(process_file, range(len(videos)))
 return list(predictions )<load_pretrained>
model = sm.OLS(target.values, new_train.astype(float))
input_size = 224
checkpoint = torch.load("/kaggle/input/deepfakes-inference-demo/resnext.pth", map_location=gpu)
 model = MyResNeXt().to(gpu)
 model.load_state_dict(checkpoint)
 _ = model.eval()
del checkpoint 
<predict_on_test>
re = model.fit()
start_time = time.time()
speedtest_videos = test_videos[:5]
predictions = predict_on_video_set(speedtest_videos, num_workers=4)
 elapsed = time.time() - start_time
print("Elapsed %f sec.Average per video: %f sec." %(elapsed, elapsed / len(speedtest_videos)) )<install_modules>
param = {'alpha':[1e-03,0.01,0.1,0.5,0.8,1], 'normalize':[True,False], 'tol':[1e-06,1e-05,5e-05,1e-04,5e-04,1e-03]}
!pip install.. /input/deepfake-xception-trained-model/pytorchcv-0.0.55-py2.py3-none-any.whl --quiet<normalization>
lasso = make_pipeline(GridSearchCV(Lasso(random_state=1),param,
cv=10, scoring = RMSLE))
class Head(nn.Module):
def __init__(self, in_f, out_f):
super().__init__()
self.f = nn.Flatten()
self.l = nn.Linear(in_f, 512)
 self.b1 = nn.BatchNorm1d(in_f)
 self.d = nn.Dropout(0.25)
 self.o = nn.Linear(512, out_f)
 self.b2 = nn.BatchNorm1d(512)
 self.r = nn.ReLU()
def forward(self, x):
x = self.f(x)
 x = self.b1(x)
 x = self.d(x)
 x = self.l(x)
 x = self.r(x)
 x = self.b2(x)
 x = self.d(x)
 out = self.o(x)
 return out
class FCN(nn.Module):
def __init__(self, base, in_f):
super().__init__()
self.base = base
self.h1 = Head(in_f, 1)
 def forward(self, x):
x = self.base(x)
 return self.h1(x )<choose_model_class>
lasso.fit(X_train,y_train )
model = get_model("xception", pretrained=False)
 model = nn.Sequential(*list(model.children())[:-1])
 model[0].final_block.pool = nn.Sequential(nn.AdaptiveAvgPool2d(( 1,1)) )<load_pretrained>
la_yhat = lasso.predict(X_val )
input_size = 150
checkpoint = torch.load('.. /input/mymodels/model_niz.pth', map_location=gpu)
 model = FCN(model, 2048 ).to(gpu)
 model.load_state_dict(checkpoint)
 _ = model.eval()
del checkpoint 
<predict_on_test>
s_lasso = rmsle_score(la_yhat,y_val)
 s_lasso
start_time = time.time()
speedtest_videos = test_videos[:5]
predictions = predict_on_video_set(speedtest_videos, num_workers=4)
 elapsed = time.time() - start_time
print("Elapsed %f sec.Average per video: %f sec." %(elapsed, elapsed / len(speedtest_videos)) )<create_dataframe>
pred_la = lasso.predict(new_test )
final_predictions = 0.5 *(np.array(predictions_resnext)+ np.array(predictions_xception))
submission_df = pd.DataFrame({'filename': test_videos, "label": final_predictions})
 plt.hist(submission_df['label'] )<save_to_csv>
param_e = {'alpha' :[0.1,1.0,10], 'max_iter' :[1000000], 'l1_ratio':[0.04,0.05],
'normalize':[True,False]}
submission_df.to_csv("submission.csv", index=False)
<import_modules>
Enet = make_pipeline(GridSearchCV(ElasticNet(random_state=1),param_e,
cv=10, scoring = RMSLE))
from IPython.display import Image<define_variables>
Enet.fit(X_train,y_train )
Image('.. /input/deepfake-kernel-data/google_cloud_compute_engine_launch_vm.png' )<define_variables>
Enet_yhat = Enet.predict(X_val )
Image('.. /input/deepfake-kernel-data/google_cloud_vm.png' )<define_variables>
s_Enet = rmsle_score(Enet_yhat,y_val)
 s_Enet
Image('.. /input/deepfake-kernel-data/lr_15e-2_epochs_42_patience_5.png' )<define_variables>
pred_Enet = Enet.predict(new_test )
Image('.. /input/deepfake-kernel-data/lr_2e-3_epochs_10_patience_5.png' )<define_variables>
param_Rf = {'min_samples_split' : [3,4,6,10], 'n_estimators' : [70,100], 'random_state': [5] }
Image('.. /input/deepfake-kernel-data/lr_2e-3_epochs_20_patience_5.png' )<define_variables>
RF = make_pipeline(GridSearchCV(RandomForestRegressor(random_state=1),param_Rf,
cv=10, scoring = RMSLE))
Image('.. /input/deepfake-kernel-data/lr_4e-3_epochs_12_patience_2.png' )<define_variables>
RF.fit(X_train,y_train )
Image('.. /input/deepfake-kernel-data/lr_4e-3_epochs_30_patience_2.png' )<define_variables>
RF_yhat = RF.predict(X_val)
 s_RF = rmsle_score(RF_yhat,y_val)
 s_RF
Image('.. /input/deepfake-kernel-data/google_cloud_vm_deepfake_training_screenshot.png' )<set_options>
pred_RF = RF.predict(new_test )
%matplotlib inline
warnings.filterwarnings("ignore" )<define_variables>
param_GB = [{'learning_rate': [1,0.1,0.01,0.001],
'n_estimators': [50, 100, 200, 500, 1000]}]
test_dir = "/kaggle/input/deepfake-detection-challenge/test_videos/"
test_videos = sorted([x for x in os.listdir(test_dir)if x[-4:] == ".mp4"])
 frame_h = 5
frame_l = 5
len(test_videos )<import_modules>
GB = make_pipeline(GridSearchCV(GradientBoostingRegressor(random_state=1),param_GB,
cv=10, scoring = RMSLE))
print("PyTorch version:", torch.__version__)
 print("CUDA version:", torch.version.cuda)
 print("cuDNN version:", torch.backends.cudnn.version() )<set_options>
GB.fit(X_train,y_train )
gpu = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 gpu<load_pretrained>
GB_yhat = GB.predict(X_val)
 s_GB = rmsle_score(GB_yhat,y_val)
 s_GB
facedet = BlazeFace().to(gpu)
 facedet.load_weights("/kaggle/input/blazeface-pytorch/blazeface.pth")
 facedet.load_anchors("/kaggle/input/blazeface-pytorch/anchors.npy")
 _ = facedet.train(False )<load_pretrained>
pred_GB = GB.predict(new_test )
frames_per_video = 64
video_reader = VideoReader()
video_read_fn = lambda x: video_reader.read_frames(x, num_frames=frames_per_video)
 face_extractor = FaceExtractor(video_read_fn, facedet )<define_variables>
param_lgb = param_grid = [{
'n_estimators': [400, 700, 1000],
'max_depth': [15,20,25],
'num_leaves': [50, 100, 200],
'min_split_gain': [0.3, 0.4],
}]
input_size = 224<normalization>
lgb = make_pipeline(GridSearchCV(LGBMRegressor(verbose_eval=False,random_state=1),param_lgb,
cv=10, scoring = RMSLE))
mean = [0.485, 0.456, 0.406]
std = [0.229, 0.224, 0.225]
normalize_transform = Normalize(mean, std )<choose_model_class>
lgb.fit(X_train,y_train )
class MyResNeXt(models.resnet.ResNet):
def __init__(self, training=True):
super(MyResNeXt, self ).__init__(block=models.resnet.Bottleneck,
layers=[3, 4, 6, 3],
groups=32,
width_per_group=4)
 self.fc = nn.Linear(2048, 1 )<load_pretrained>
lgb_yhat = lgb.predict(X_val)
 s_lgb = rmsle_score(lgb_yhat,y_val)
 s_lgb
checkpoint = torch.load("/kaggle/input/deepfakes-inference-demo/resnext.pth", map_location=gpu)
 model = MyResNeXt().to(gpu)
 model.load_state_dict(checkpoint)
 _ = model.eval()
del checkpoint<predict_on_test>
pred_lgb = lgb.predict(new_test )
def predict_on_video(video_path, batch_size):
try:
faces = face_extractor.process_video(video_path)
 face_extractor.keep_only_best_face(faces)
 if len(faces)> 0:
x = np.zeros(( batch_size, input_size, input_size, 3), dtype=np.uint8)
 n = 0
for frame_data in faces:
for face in frame_data["faces"]:
resized_face = isotropically_resize_image(face, input_size)
 resized_face = make_square_image(resized_face)
 if n < batch_size:
x[n] = resized_face
n += 1
else:
print("WARNING: have %d faces but batch size is %d" %(n, batch_size))
if n > 0:
x = torch.tensor(x, device=gpu ).float()
x = x.permute(( 0, 3, 1, 2))
for i in range(len(x)) :
x[i] = normalize_transform(x[i] / 255.)
with torch.no_grad() :
y_pred = model(x)
 y_pred = torch.sigmoid(y_pred.squeeze())
 return y_pred[:n].mean().item()
except Exception as e:
print("Prediction error on video %s: %s" %(video_path, str(e)))
 return 0.5<predict_on_test>
list_scores = [s_lasso, s_Enet, s_RF,s_GB,s_lgb]
list_regressors = ['Lasso','Enet','RF','GB','lgb']
def predict_on_video_set(videos, num_workers):
def process_file(i):
filename = videos[i]
y_pred = predict_on_video(os.path.join(test_dir, filename), batch_size=frames_per_video)
 return y_pred
with ThreadPoolExecutor(max_workers=num_workers)as ex:
predictions = ex.map(process_file, range(len(videos)))
 return list(predictions )<define_variables>
predictions = {'Lasso': pred_la,
'ElaNet': pred_Enet,
'RF': pred_RF,
'GB': pred_GB,
'lgb' : pred_lgb
}
speed_test = False<predict_on_test>
df_predictions = pd.DataFrame(data=predictions)
 df_predictions.corr()
if speed_test:
start_time = time.time()
speedtest_videos = test_videos[:5]
predictions = predict_on_video_set(speedtest_videos, num_workers=4)
 elapsed = time.time() - start_time
print("Elapsed %f sec.Average per video: %f sec." %(elapsed, elapsed / len(speedtest_videos)) )<predict_on_test>
RF.fit(new_train,target )
predictions = predict_on_video_set(test_videos, num_workers=4 )<save_to_csv>
log_pred=RF.predict(new_test)
 predictions=np.exp(log_pred)-1
submission_df_resnext = pd.DataFrame({"filename": test_videos, "label": predictions})
 submission_df_resnext.to_csv("submission_resnext.csv", index=False )<install_modules>
sub = pd.DataFrame()
sub['datetime'] = test['datetime']
sub['count'] = predictions
sub.head()
!pip install.. /input/deepfake-xception-trained-model/pytorchcv-0.0.55-py2.py3-none-any.whl --quiet<define_variables>
sub.to_csv('submission.csv', index=False )
test_dir = "/kaggle/input/deepfake-detection-challenge/test_videos/"
test_videos = sorted([x for x in os.listdir(test_dir)if x[-4:] == ".mp4"])
 len(test_videos )<set_options>
sub.to_csv('submission.csv', index=False )
gpu = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 gpu<load_pretrained>
lgb.fit(new_train,target )
facedet = BlazeFace().to(gpu)
 facedet.load_weights("/kaggle/input/blazeface-pytorch/blazeface.pth")
 facedet.load_anchors("/kaggle/input/blazeface-pytorch/anchors.npy")
 _ = facedet.train(False )<load_pretrained>
log_pred_lgb=lgb.predict(new_test)
 predictions_lgb=np.exp(log_pred_lgb)-1
frames_per_video = 64
video_reader = VideoReader()
video_read_fn = lambda x: video_reader.read_frames(x, num_frames=frames_per_video)
 face_extractor = FaceExtractor(video_read_fn, facedet )<define_variables>
sub = pd.DataFrame()
sub['datetime'] = test['datetime']
sub['count'] = predictions_lgb
sub.head()
input_size = 150<normalization>
ensemble =(0.6*predictions + 0.4*predictions_lgb )
mean = [0.485, 0.456, 0.406]
std = [0.229, 0.224, 0.225]
normalize_transform = Normalize(mean, std )<choose_model_class>
sub = pd.DataFrame()
sub['datetime'] = test['datetime']
sub['count'] = ensemble
sub.head()
model = get_model("xception", pretrained=False)
 model = nn.Sequential(*list(model.children())[:-1])
 class Pooling(nn.Module):
def __init__(self):
super(Pooling, self ).__init__()
self.p1 = nn.AdaptiveAvgPool2d(( 1,1))
self.p2 = nn.AdaptiveMaxPool2d(( 1,1))
def forward(self, x):
x1 = self.p1(x)
 x2 = self.p2(x)
 return(x1+x2)* 0.5
model[0].final_block.pool = nn.Sequential(nn.AdaptiveAvgPool2d(( 1,1)))
 class Head(torch.nn.Module):
def __init__(self, in_f, out_f):
super(Head, self ).__init__()
self.f = nn.Flatten()
self.l = nn.Linear(in_f, 512)
 self.d = nn.Dropout(0.5)
 self.o = nn.Linear(512, out_f)
 self.b1 = nn.BatchNorm1d(in_f)
 self.b2 = nn.BatchNorm1d(512)
 self.r = nn.ReLU()
def forward(self, x):
x = self.f(x)
 x = self.b1(x)
 x = self.d(x)
 x = self.l(x)
 x = self.r(x)
 x = self.b2(x)
 x = self.d(x)
 out = self.o(x)
 return out
class FCN(torch.nn.Module):
def __init__(self, base, in_f):
super(FCN, self ).__init__()
self.base = base
self.h1 = Head(in_f, 1)
 def forward(self, x):
x = self.base(x)
 return self.h1(x)
 net = []
model = FCN(model, 2048)
 model = model.cuda()
model.load_state_dict(torch.load('.. /input/deepfake-kernel-data/model_50epochs_lr0001_patience5_factor01_batchsize32.pth'))
net.append(model )<predict_on_test>
sub.to_csv('submission.csv', index=False )
def predict_on_video(video_path, batch_size):
try:
faces = face_extractor.process_video(video_path)
 face_extractor.keep_only_best_face(faces)
 if len(faces)> 0:
x = np.zeros(( batch_size, input_size, input_size, 3), dtype=np.uint8)
 n = 0
for frame_data in faces:
for face in frame_data["faces"]:
resized_face = isotropically_resize_image(face, input_size)
 resized_face = make_square_image(resized_face)
 if n < batch_size:
x[n] = resized_face
n += 1
else:
print("WARNING: have %d faces but batch size is %d" %(n, batch_size))
if n > 0:
x = torch.tensor(x, device=gpu ).float()
x = x.permute(( 0, 3, 1, 2))
for i in range(len(x)) :
x[i] = normalize_transform(x[i] / 255.)
with torch.no_grad() :
y_pred = model(x)
 y_pred = torch.sigmoid(y_pred.squeeze())
 return y_pred[:n].mean().item()
except Exception as e:
print("Prediction error on video %s: %s" %(video_path, str(e)))
 return 0.5<predict_on_test>
sub.to_csv('submission.csv', index=False )