Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<save_to_csv>
%matplotlib inline
simple_accurate_99 = pd.read_csv(dfk.iloc[5, 2])
 sub['floor'] = simple_accurate_99['floor'].values
sub.to_csv('i7274cm99.csv', index=False)
 i7274cm99 = sub<feature_engineering>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )
sub = pd.read_csv(dfk.iloc[9, 2])
 tmp = sub['site_path_timestamp'].apply(lambda s : pd.Series(s.split('_')))
 sub['site'] = tmp[0]
sub['path'] = tmp[1]
sub['timestamp'] = tmp[2].astype(float)
 processes = multiprocessing.cpu_count()
with multiprocessing.Pool(processes=processes)as pool:
dfs = pool.imap_unordered(correct_path, sub.groupby('path'))
dfs = tqdm(dfs)
 dfs = list(dfs)
 sub = pd.concat(dfs ).sort_values('site_path_timestamp' )<save_to_csv>
y_train = train["label"]
g = sns.countplot(y_train)
 y_train.value_counts()
X_train = train.drop(labels = ["label"],axis = 1)
 del train
simple_accurate_99 = pd.read_csv(dfk.iloc[5, 2])
 sub['floor'] = simple_accurate_99['floor'].values
sub.to_csv('j7285cm99.csv', index=False)
 j7285cm99 = sub<create_dataframe>
X_train = X_train / 255.0
test = test / 255.0
gfk = pd.DataFrame({
'Kernel ID' : ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J'],
'Before Score': [9.773, 9.530, 8.500, 8.418, 8.333, 8.073, 7.745, 7.661, 7.274, 7.285],
'After Score' : [7.110, 6.674, 6.290, 6.189, 6.077, 6.062, 5.995, 5.694, 5.471, 5.847],
'File Name' : ['a9773cm99.csv', 'b9530cm99.csv', 'c8500cm99.csv', 'd8418cm99.csv', 'e8333cm99.csv', 'f8073cm99.csv', 'g7745cm99.csv', 'h7661cm99.csv', 'i7274cm99.csv', 'j7285cm99.csv']
})
 gfk<save_to_csv>
y_train = to_categorical(y_train, num_classes = 10 )
sub = i7274cm99
sub.to_csv("submission.csv", index=False)
 a9773cm99.to_csv("a9773cm99.csv", index=False)
 b9530cm99.to_csv("b9530cm99.csv", index=False)
 c8500cm99.to_csv("c8500cm99.csv", index=False)
 d8418cm99.to_csv("d8418cm99.csv", index=False)
 e8333cm99.to_csv("e8333cm99.csv", index=False)
 f8073cm99.to_csv("f8073cm99.csv", index=False)
 g7745cm99.to_csv("g7745cm99.csv", index=False)
 h7661cm99.to_csv("h7661cm99.csv", index=False)
 i7274cm99.to_csv("i7274cm99.csv", index=False)
 j7285cm99.to_csv("j7285cm99.csv", index=False)
 !ls<define_variables>
random_seed = 2
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size = 0.15, random_state=random_seed )
%matplotlib inline
dfk = pd.DataFrame({
'Kernel ID': ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M'],
'Score': [14.763, 13.168, 11.600, 10.959, 9.773, 9.530, 8.500, 8.418, 8.333, 8.309, 8.073, 7.745, 7.661],
'File Path': ['.. /input/indoor14763/Indoor14763.csv', '.. /input/indoor13168/Indoor13168.csv', '.. /input/indoor11600/Indoor11600.csv', '.. /input/indoor10959/Indoor10959.csv', '.. /input/indoor9773/Indoor9773.csv', '.. /input/indoor9530/Indoor9530.csv', '.. /input/indoor8500/Indoor8500.csv', '.. /input/indoor8418/Indoor8418.csv', '.. /input/indoor8333/Indoor8333.csv', '.. /input/indoor8309/Indoor8309.csv', '.. /input/indoor8073/Indoor8073.csv', '.. /input/indoornav7745sub/submission.csv', '.. /input/indoor7661/Indoor7661.csv']
})
 dfk<prepare_x_and_y>
model = models.Sequential()
ff1x = 0.50
ff1y = 0.50
ff2x = 0.95
ff2y = 0.95
ff3x = 0.90
ff3y = 0.90
main = pd.read_csv(dfk.iloc[12, 2])
 support = pd.read_csv(dfk.iloc[11, 2])
 rx , ry = [] , []
generated = main.copy()
mx , my , mz = [] , [] , []
mx = main.iloc[:, 2].tolist()
my = main.iloc[:, 3].tolist()
mz = main.iloc[:, 1].tolist()
sx , sy = [] , []
sx = support.iloc[:, 2].tolist()
sy = support.iloc[:, 3].tolist()
kx = [[],[],[],[],[],[],[],[],[],[],[]]
ky = [[],[],[],[],[],[],[],[],[],[],[]]
for n in range(11):
kernel = pd.read_csv(dfk.iloc[n, 2])
 kx[n] = kernel.iloc[:, 2].tolist()
ky[n] = kernel.iloc[:, 3].tolist()
for j in range(len(main)) :
ex =(mx[j] * ff1x)+(sx[j] *(1.- ff1x))
ey =(my[j] * ff1y)+(sy[j] *(1.- ff1y))
if(mx[j] >= sx[j]):
a = mx[j]
b = sx[j]
else:
a = sx[j]
b = mx[j]
if(my[j] >= sy[j]):
c = my[j]
d = sy[j]
else:
c = sy[j]
d = my[j]
for k in range(5):
if(( b <= kx[k][j] <= a)and(d <= ky[k][j] <= c)) :
ex =(ex * ff2x)+(kx[k][j] *(1.- ff2x))
ey =(ey * ff2y)+(ky[k][j] *(1.- ff2y))
print(f'Row:{j}\t Kernel Number:{k}\t Floor:{mz[j]}')
 print(f'Main:({mx[j]},{my[j]})
 Support:({sx[j]},{sy[j]})
 Kernel:({kx[k][j]},{ky[k][j]})
 Result:({ex},{ey})
 ')
 for k in range(5, 11):
if(( b <= kx[k][j] <= a)and(d <= ky[k][j] <= c)) :
ex =(ex * ff3x)+(kx[k][j] *(1.- ff3x))
ey =(ey * ff3y)+(ky[k][j] *(1.- ff3y))
print(f'Row:{j}\t Kernel Number:{k}\t Floor:{mz[j]}')
 print(f'Main:({mx[j]},{my[j]})
 Support:({sx[j]},{sy[j]})
 Kernel:({kx[k][j]},{ky[k][j]})
 Result:({ex},{ey})
 ')
 rx.append(ex)
 ry.append(ey)
 generated.iloc[:, 2] = rx
generated.iloc[:, 3] = ry 
<import_modules>
model.add(Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same', activation ='relu', input_shape =(28,28,1)) )
import json
import matplotlib.pylab as plt 
<save_to_csv>
model.add(Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same', activation ='relu'))
sub = generated
sub.to_csv("submission.csv", index=False)
<set_options>
model.add(layers.MaxPooling2D(( 2, 2)) )
sys.path.append('.. /input/pytorch-images-seresnet')
 warnings.filterwarnings('ignore')
 device = torch.device('cuda' if torch.cuda.is_available() else 'cpu' )<define_variables>
model.add(Dropout(0.25))
IMAGE_SIZE = 640
BATCH_SIZE = 128
TEST_PATH = '.. /input/ranzcr-clip-catheter-line-classification/test'
MODEL_PATH = '.. /input/seresnet152d-cv9615/seresnet152d_320_CV96.15.pth'<load_from_csv>
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same', activation ='relu'))
test = pd.read_csv('.. /input/ranzcr-clip-catheter-line-classification/sample_submission.csv' )<categorify>
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same', activation ='relu'))
def get_transforms() :
return Compose([
Resize(IMAGE_SIZE, IMAGE_SIZE),
Normalize(
),
ToTensorV2() ,
] )<define_search_model>
model.add(layers.MaxPooling2D(( 2, 2)) )
class SeResNet152D(nn.Module):
def __init__(self, model_name='seresnet152d_320'):
super().__init__()
self.model = timm.create_model(model_name, pretrained=False)
 n_features = self.model.fc.in_features
self.model.global_pool = nn.Identity()
self.model.fc = nn.Identity()
self.pooling = nn.AdaptiveAvgPool2d(1)
 self.fc = nn.Linear(n_features, 11)
 def forward(self, x):
bs = x.size(0)
 features = self.model(x)
 pooled_features = self.pooling(features ).view(bs, -1)
 output = self.fc(pooled_features)
 return output<categorify>
model.add(Dropout(0.25))
def inference(models, test_loader, device):
tk0 = tqdm(enumerate(test_loader), total=len(test_loader))
probs = []
for i,(images)in tk0:
images = images.to(device)
 avg_preds = []
for model in models:
with torch.no_grad() :
y_preds1 = model(images)
 y_preds2 = model(images.flip(-1))
y_preds =(y_preds1.sigmoid().to('cpu' ).numpy() + y_preds2.sigmoid().to('cpu' ).numpy())/ 2
avg_preds.append(y_preds)
 avg_preds = np.mean(avg_preds, axis=0)
 probs.append(avg_preds)
 probs = np.concatenate(probs)
 return probs<choose_model_class>
model.add(Flatten())
 model.add(Dense(256, activation = "relu"))
model = SeResNet152D()
model.load_state_dict(torch.load(MODEL_PATH)['model'])
 model.eval()
models = [model.to(device)]<load_pretrained>
model.add(BatchNormalization() )
test_dataset = TestDataset(test, transform=get_transforms())
 test_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=False,
num_workers=4 , pin_memory=True)
 predictions = inference(models, test_loader, device )<save_to_csv>
model.add(Dense(10, activation = "softmax"))
target_cols = test.iloc[:, 1:12].columns.tolist()
test[target_cols] = predictions
test[['StudyInstanceUID'] + target_cols].to_csv('submission.csv', index=False)
 test.head()<import_modules>
my_optimizer = RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0 )
warnings.filterwarnings('ignore')
 %config InlineBackend.figure_format = 'retina'
%matplotlib inline
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
input_dir = "/kaggle/input/house-prices-advanced-regression-techniques/" 
<load_from_csv>
model.compile(optimizer=my_optimizer,
loss='categorical_crossentropy',
metrics=['accuracy'] )
train = pd.read_csv(f"{input_dir}/train.csv")
 test = pd.read_csv(f"{input_dir}/test.csv")
 print("Train shape:{}".format(train.shape))
print("Test shape:{}
".format(test.shape))
train.head()<drop_column>
datagen = ImageDataGenerator(rotation_range=10,
zoom_range=0.1,
width_shift_range=0.1,
height_shift_range=0.1,
fill_mode='nearest')
 datagen.fit(X_train )
train.drop(['Id'], axis=1, inplace=True)
 test.drop(['Id'], axis=1, inplace=True)
 train.head()<concatenate>
learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001 )
features = pd.concat([train_feat, test_feat] ).reset_index(drop=True)
 print(features.shape)
 features.head()<categorify>
history = model.fit_generator(datagen.flow(X_train, y_train, batch_size=86),
epochs=100, validation_data =(X_val, y_val),
verbose = 2, steps_per_epoch=X_train.shape[0] // 86,
callbacks=[learning_rate_reduction] )
fill_with_none =('PoolQC', 'Alley', 'MiscFeature', 'Fence', 'FireplaceQu', 'GarageFinish', 'GarageQual', 'GarageCond', 'GarageType', 'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2', 'MasVnrType')
 fill_with_mode =('Utilities', 'Functional')
 fill_with_0 =('GarageYrBlt', 'MasVnrArea', 'BsmtFullBath', 'BsmtHalfBath')
 for col in fill_with_none:
features[col]=features[col].fillna('None')
 for col in fill_with_mode:
features[col]=features[col].fillna(features[col].mode() [0])
 for col in fill_with_0:
features[col]=features[col].fillna(0)
 for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
features[col] = features[col].fillna(0)
 for col in ['GarageType', 'GarageFinish', 'GarageQual', 'GarageCond']:
features[col] = features[col].fillna('None')
 for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
features[col] = features[col].fillna('None')
 features['Exterior1st'] = features['Exterior1st'].fillna(features['Exterior1st'].mode() [0])
 features['Exterior2nd'] = features['Exterior2nd'].fillna(features['Exterior2nd'].mode() [0])
 features['SaleType'] = features['SaleType'].fillna(features['SaleType'].mode() [0] )<groupby>
test_loss, test_acc = model.evaluate(X_val, y_val)
 test_acc
features['MSZoning'] = features.groupby('MSSubClass')['MSZoning'].transform(lambda x: x.fillna(x.mode() [0]))<categorify>
results = model.predict(test)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label" )
features['LotFrontage'] = features.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.median()))
numeric_dtypes = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
numerics = []
for i in features.columns:
if features[i].dtype in numeric_dtypes:
numerics.append(i)
 features.update(features[numerics].fillna(0))
numerics[1:10]<data_type_conversions>
submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("cnn_mnist_datagen.csv",index=False )