Kaggle competition: Digit Recognizer
cond1 = data['workingday']== 'Yes'
cond2 = data['hour'].isin([7,8,9,17,18,19])
 cond3 = data['hour'].isin([10,11,12,13,14,15,16])
 cond4 = data['hour'].isin([0,1,2,3,4,5,6,20,21,22,23])
 conditions =[~cond1,cond1&cond2,cond1&cond3,cond1&cond4]
vals = ['No','High','Medium','Low']
data['Map demand'] = np.select(conditions,vals)
<categorify>
import os
from pathlib import Path
from PIL import Image
data=additional(data )<filter>
train = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')
 test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv' )
cat_features = data.columns[data.dtypes=='object']
cat_features=list(cat_features)
 cat_features<count_values>
def create_image_from_row(digit, dest_path):
mat = digit.reshape(28,28)
 mat = mat.astype(np.uint8)
 img = Image.fromarray(mat)
 img.save(dest_path )
def count_category(name):
print(name)
 print(data[name].value_counts())
 print("-------------------------------------------------")
 for i in cat_features:
count_category(i )<groupby>
for index, row in train.iterrows() :
label,digit = row[0], row[1:]
directory = images_train/str(label)
 filename = f"{index}.jpg"
dest_path = directory/filename
digit = digit.values
create_image_from_row(digit, dest_path )
count_bikes_by_hour = data.groupby("hour")["Number of rented bikes"].sum()
print(count_bikes_by_hour )<groupby>
for index, row in test.iterrows() :
filename = f"{index}.jpg"
dest_path = images_test/filename
digit = row.values
create_image_from_row(digit, dest_path )
count_bikes_by_Holiday = data.groupby("holiday")["Number of rented bikes"].sum()
print(count_bikes_by_Holiday )<groupby>
print('A few samples of the training images.')
 displayTrainingData()
count_bikes_by_funcday = data.groupby("workingday")["Number of rented bikes"].sum()
print(count_bikes_by_funcday )<groupby>
from fastai.vision import *
count_bikes_by_Season = data.groupby("season")["Number of rented bikes"].sum()
print(count_bikes_by_Season )<feature_engineering>
data = ImageDataBunch.from_folder(images, train='train', test='test', valid_pct=0.2, bs=32,
ds_tfms=get_transforms() , size=224, num_workers=4 ).normalize(imagenet_stats )
def rolling_mean(df,cols,window=3):
for col in cols:
new_col = col+'_rolled'
df[new_col] = df[col].rolling(window=window ).mean()
return df 
<drop_column>
data.show_batch(rows=3, figsize=(7,8))
data = rolling_mean(data,['Temperature(C)','Wind Speed(m/s)',
'humidity(%)','weather'])
<categorify>
data.classes, data.c, len(data.train_ds), len(data.valid_ds )
def shifting(df,cols):
for col in cols:
new_col = col+'_shifted'
df[new_col] = df[col].shift(periods=-1)
 return df
data = shifting(data,['Temperature(C)','Wind Speed(m/s)','weather', 'humidity(%)'])
 def shifting_1(df,cols):
for col in cols:
new_col = col+'_shifted1'
df[new_col] = df[col].shift(periods=-2)
 return df
data = shifting_1(data,['Temperature(C)','Wind Speed(m/s)','weather', 'humidity(%)'])
 def shifting_2(df,cols):
for col in cols:
new_col = col+'_shifted2'
df[new_col] = df[col].shift(periods=-3)
 return df
data = shifting_2(data,['Temperature(C)','Wind Speed(m/s)','weather', 'humidity(%)'] )<data_type_conversions>
learn = cnn_learner(data, models.resnet18, metrics=error_rate )
data=data.fillna(data.mean() )<categorify>
learn.fit_one_cycle(4 )
def conv_cat_features(df):
df['holiday'].replace({"No Holiday":1, "Holiday":0}, inplace=True)
 df['workingday'].replace({"Yes":1, "No":0}, inplace=True)
 data_cat_transformed = pd.get_dummies(df, drop_first=True)
 return data_cat_transformed<categorify>
learn.save('stage-1' )
data_cat_transformed=conv_cat_features(data)
 print("{} total features after one-hot encoding.".format(len(data_cat_transformed.columns)))
 print(data_cat_transformed.columns )<split>
learn.lr_find()
X = data_cat_transformed.drop('Number of rented bikes', axis=1)
 y = data_cat_transformed['Number of rented bikes']
def custom_train_valid_split(data, cutoff_day=15):
train = data[data['day'] <= cutoff_day]
valid = data[data['day'] > cutoff_day]
return train,valid
train,valid = custom_train_valid_split(data, cutoff_day=15)
 train_idx = train.index
valid_idx = valid.index
myCViterator = []
myCViterator.append(( train_idx, valid_idx))<compute_test_metric>
learn.fit_one_cycle(2, max_lr=slice(3e-5,3e-3))
def RMSLE(y_train,y_pred):
return np.sqrt(mean_squared_log_error(y_train, y_pred))<load_from_csv>
learn.save('stage-2' )
test_df=pd.read_csv('.. /input/bike-sharing-demand/test.csv')
 test_df_copy=test_df.copy()<rename_columns>
learn.load('stage-2');
test_df=test_df.rename(columns={col_names[5]: "Temperature(C)",col_names[6]:"Dew Temperature(C)",col_names[7]: "humidity(%)",
col_names[8]:"Wind Speed(m/s)"
} )<data_type_conversions>
interp = ClassificationInterpretation.from_learner(learn )
test_df=convert_date(test_df )<feature_engineering>
losses,idxs = interp.top_losses()
cond1 = test_df['workingday']=='Yes'
cond2 = test_df['hour'].isin([7,8,9,17,18,19])
 cond3 = test_df['hour'].isin([10,11,12,13,14,15,16])
 cond4 = test_df['hour'].isin([0,1,2,3,4,5,6,20,21,22,23])
 conditions =[~cond1,cond1&cond2,cond1&cond3,cond1&cond4]
vals = ['No','High','Medium','Low']
test_df['Map demand'] = np.select(conditions,vals )<drop_column>
class_score, y = learn.get_preds(DatasetType.Test )
test_df=test_df.drop(['datetime',],axis=1)
<concatenate>
probabilities = class_score[0].tolist()
[f"{index}: {probabilities[index]}" for index in range(len(probabilities)) ]
test_df=additional(test_df )<drop_column>
class_score = np.argmax(class_score, axis=1 )
test_df = rolling_mean(test_df,['Temperature(C)','Wind Speed(m/s)','weather', 'humidity(%)'])
 test_df = shifting(test_df,['Temperature(C)','Wind Speed(m/s)','weather', 'humidity(%)'])
 test_df = shifting_1(test_df,['Temperature(C)','Wind Speed(m/s)','weather', 'humidity(%)'])
 test_df = shifting_2(test_df,['Temperature(C)','Wind Speed(m/s)','weather', 'humidity(%)'])
 test_df=test_df.fillna(test_df.mean() )<feature_engineering>
INPUT = Path(".. /input/digit-recognizer")
 TEST = Path(".. /test" )
test_cat_transformed=conv_cat_features(test_df )<init_hyperparams>
ImageId = [os.path.splitext(path)[0] for path in os.listdir(images_test)]
ImageId = [int(path)for path in ImageId]
ImageId = [ID+1 for ID in ImageId]
<train_on_grid><EOS> 
submission = pd.DataFrame({
"ImageId": ImageId,
"Label": class_score
})
 submission.to_csv("submission.csv", index=False)
 display(submission.head(3))
display(submission.tail(3))