Kaggle competition: QSTP - Deep Learning 2019
<SOS> metric: categorizationaccuracy Kaggle data source: qstp-deep-learning-2019<load_from_csv>
%matplotlib inline
df_train = pd.read_csv('.. /input/train.csv')
 test_path = '.. /input/test.csv'
df_test = pd.read_csv(test_path)
 df_test.drop(['ID_code'], axis=1, inplace=True)
 df_test = df_test.values
unique_samples = []
unique_count = np.zeros_like(df_test)
 for feature in tqdm(range(df_test.shape[1])) :
_, index_, count_ = np.unique(df_test[:, feature], return_counts=True, return_index=True)
 unique_count[index_[count_ == 1], feature] += 1
real_samples_indexes = np.argwhere(np.sum(unique_count, axis=1)> 0)[:, 0]
synthetic_samples_indexes = np.argwhere(np.sum(unique_count, axis=1)== 0)[:, 0]
print(len(real_samples_indexes))
print(len(synthetic_samples_indexes)) 
<load_from_csv>
df = pd.read_csv('.. /input/train.csv')
 df.head()
train, validate = np.split(df, [int (.8*len(df)) ])
 validate.index = range(len(validate))
df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv' )<load_from_csv>
class ImageDataset(Dataset):
def __init__(self, data, root_dir, transform=None):
self.df = data
self.dir = root_dir
self.transform = transform
def __len__(self):
return len(self.df)
 def __getitem__(self, index):
img_path = os.path.join(self.dir, self.df['Id'][index])
 image = Image.open(img_path ).convert('RGB')
 label = np.array(self.df['Category'][index])
 if self.transform:
image = self.transform(image)
 sample =(image, label)
 return sample
df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv')
 def findmeancounts(bins,values,x):
for i in range(len(bins)) :
if x < bins[i]:
return values[i]
return values[-1]
def magic2() :
feats = ["var_"+str(i)for i in range(200)]
df = pd.concat([df_train,df_test.ix[real_samples_indexes]])
 for feat in feats:
temp = df[feat].value_counts(dropna = True)
 df_train[feat+"vc"] = df_train[feat].map(temp ).map(lambda x:min(10,x)).astype(np.uint8)
 df_test[feat+"vc"] = df_test[feat].map(temp ).map(lambda x:min(10,x)).astype(np.uint8)
 print(feat,temp.shape[0],df_train[feat+"vc"].map(lambda x:int(x>2)).sum() ,df_train[feat+"vc"].map(lambda x:int(x>3)).sum())
 df_train[feat+"sum"] =(( df_train[feat] - df[feat].mean())* df_train[feat+"vc"].map(lambda x:int(x>1)) ).astype(np.float32)
 df_test[feat+"sum"] =(( df_test[feat] - df[feat].mean())* df_test[feat+"vc"].map(lambda x:int(x>1)) ).astype(np.float32)
 df_train[feat+"sum2"] =(( df_train[feat])* df_train[feat+"vc"].map(lambda x:int(x>2)) ).astype(np.float32)
 df_test[feat+"sum2"] =(( df_test[feat])* df_test[feat+"vc"].map(lambda x:int(x>2)) ).astype(np.float32)
 df_train[feat+"sum3"] =(( df_train[feat])* df_train[feat+"vc"].map(lambda x:int(x>4)) ).astype(np.float32)
 df_test[feat+"sum3"] =(( df_test[feat])* df_test[feat+"vc"].map(lambda x:int(x>4)) ).astype(np.float32)
 magic2()
df_train.iloc[:5,:]<categorify>
transform = transforms.Compose([transforms.Resize(256),
transforms.RandomRotation(24),
transforms.ColorJitter(brightness=0.8, contrast=0.2),
transforms.CenterCrop(224),
transforms.RandomHorizontalFlip() ,
transforms.ToTensor() ,
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])])
 trainset = ImageDataset(data=train, root_dir='.. /input/data/data', transform=transform)
 valset = ImageDataset(data=validate, root_dir='.. /input/data/data', transform=transform)
 datasets = {'train' : trainset, 'val': valset}
dataloaders = {x : DataLoader(datasets[x], batch_size=32, num_workers=4, shuffle=True)
 for x in ['train', 'val']}
def augment(x,y,t=2):
xs,xn = [],[]
for i in range(t//2):
mask = y==0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 featnum = x1.shape[1]//200 - 1
for c in range(200):
np.random.shuffle(ids)
 x1[:,[c] + [200 + featnum * c + idc for idc in range(featnum)]] = x1[ids][:,[c] + [200 + featnum * c + idc for idc in range(featnum)]]
xn.append(x1)
 for i in range(t):
mask = y>0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 featnum = x1.shape[1]//200 - 1
for c in range(200):
np.random.shuffle(ids)
 x1[:,[c] + [200 + featnum * c + idc for idc in range(1)]] = x1[ids][:,[c] + [200 + featnum * c + idc for idc in range(1)]]
xs.append(x1)
 xs = np.vstack(xs)
 xn = np.vstack(xn)
 ys = np.ones(xs.shape[0])
 yn = np.zeros(xn.shape[0])
 x = np.vstack([x,xs,xn])
 y = np.concatenate([y,ys,yn])
 return x,y
features = [col for col in df_train.columns if col not in ['target', 'ID_code']] 
<groupby>
gpu = torch.cuda.is_available()
if gpu:
print("CUDA is available.Training on GPU")
 else:
print("CUDA unavailable.Training on CPU" )
print(df_train['target'].value_counts())
 print(df_train.groupby('var_80vc')['target'].value_counts())
 print(df_train.groupby('var_81vc')['target'].value_counts())
 print(df_train.groupby('var_82vc')['target'].value_counts())
<init_hyperparams>
model = models.resnet152(pretrained=True)
 num_ft = model.fc.in_features
model.fc = nn.Sequential(nn.Linear(num_ft, 67),
nn.Dropout2d(p=0.2))
model.cuda()
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters() , lr=0.001, momentum=0.9)
 scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.1 )
lgb_params = {
"objective" : "binary",
"metric" : "auc",
"boosting": 'gbdt',
"max_depth" : -1,
"num_leaves" : 15,
"learning_rate" : 0.01,
"bagging_freq": 5,
"bagging_fraction" : 0.6,
"feature_fraction" : 0.05,
"min_data_in_leaf": 50,
"min_sum_heassian_in_leaf": 10,
"tree_learner": "serial",
"boost_from_average": "false",
"lambda_l1" : 1.,
"bagging_seed" : 2007,
"verbosity" : 1,
"seed": 2007
}
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=2007)
 oof = df_train[['ID_code', 'target']]
oof['predict'] = 0
predictions = np.zeros(( df_test.shape[0],5))
val_aucs = []
feature_importance_df = pd.DataFrame()
features = [col for col in df_train.columns if col not in ['target', 'ID_code']]
X_test = df_test[features].values
for fold,(trn_idx, val_idx)in enumerate(skf.split(df_train, df_train['target'])) :
X_train, y_train = df_train.iloc[trn_idx][features], df_train.iloc[trn_idx]['target']
X_valid, y_valid = df_train.iloc[val_idx][features], df_train.iloc[val_idx]['target']
N = 1
p_valid,yp = 0,0
for i in range(N):
X_t, y_t = augment(X_train.values, y_train.values)
 weights = np.array([0.8] * X_t.shape[0])
 weights[:X_train.shape[0]] = 1.0
print(X_t.shape)
 trn_data = lgb.Dataset(X_t, label=y_t, weight = weights)
 val_data = lgb.Dataset(X_valid, label=y_valid)
 evals_result = {}
lgb_clf = lgb.train(lgb_params,
trn_data,
100000,
valid_sets = [trn_data, val_data],
early_stopping_rounds=5000,
verbose_eval = 1000,
evals_result=evals_result 
)
 p_valid += lgb_clf.predict(X_valid)
 yp += lgb_clf.predict(X_test)
 fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = features
fold_importance_df["importance"] = lgb_clf.feature_importance()
fold_importance_df["fold"] = fold + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 oof['predict'][val_idx] = p_valid/N
val_score = roc_auc_score(y_valid, p_valid)
 val_aucs.append(val_score)
 predictions[:,fold] = yp/N<compute_train_metric>
def train_model(model, criterion, optimizer, scheduler, num_epochs=20):
since = time.time()
best_acc = 0.0
best_model_state = copy.deepcopy(model.state_dict())
 for epoch in range(num_epochs):
epoch += 1
print("Epoch : {}/{}".format(epoch, num_epochs))
print("-"*10)
 for phase in ['train', 'val']:
if phase is 'train':
scheduler.step()
model.train()
else:
model.eval()
running_loss = 0.0
running_correct = 0
for inputs, labels in dataloaders[phase]:
if gpu:
inputs = Variable(inputs.cuda())
 labels = Variable(labels.cuda())
 else:
inputs = Variable(inputs)
 labels = Variable(labels)
 optimizer.zero_grad()
with torch.set_grad_enabled(phase is 'train'):
outputs = model(inputs)
 _, predicted = torch.max(outputs.data, 1)
 loss = criterion(outputs, labels)
 if phase is 'train':
loss.backward()
optimizer.step()
running_loss += loss.item() * inputs.size(0)
 running_correct += torch.sum(predicted.cpu() == labels.cpu())
 epoch_loss = running_loss / len(datasets[phase])
 epoch_acc = running_correct.double() / len(datasets[phase])
 print("{}: Loss: {:.4f}, Accuracy: {:.4f}".format(phase, epoch_loss, epoch_acc))
if phase == 'val' and epoch_acc > best_acc:
best_acc = epoch_acc
best_model_state = copy.deepcopy(model.state_dict())
 print()
time_elapsed = time.time() - since
print("Training finished in {} seconds!".format(time_elapsed))
print("Best eval accuracy: {:.4f}".format(100*best_acc))
model.load_state_dict(best_model_state)
 return model
mean_auc = np.mean(val_aucs)
 std_auc = np.std(val_aucs)
 all_auc = roc_auc_score(oof['target'], oof['predict'])
 print("Mean auc: %.9f, std: %.9f.All auc: %.9f." %(mean_auc, std_auc, all_auc))
cols =(feature_importance_df[["feature", "importance"]] 
.groupby("feature")
.mean() 
.sort_values(by="importance", ascending=False)[:1000].index)
 print(cols[:50])
 print(cols[-50:])
 best_features = feature_importance_df.loc[feature_importance_df.feature.isin(cols)] 
<save_to_csv>
model = train_model(model, criterion, optimizer, scheduler, 25 )
sub_df = pd.DataFrame({"ID_code":df_test["ID_code"].values})
 sub_df["target"] = np.mean(predictions,axis = 1)
 sub_df.to_csv("lgb_submission.csv", index=False )<set_options>
testset = ImageDataset(data=pd.read_csv('.. /input/sample_sub.csv'), root_dir='.. /input/data/data/', transform=transform)
 testloader = DataLoader(testset, batch_size=32, num_workers=4, shuffle=False )
%matplotlib inline
warnings.filterwarnings(action="ignore")
 input_path1 = '.. /input/house-prices-advanced-regression-techniques/'
input_path2 = '.. /input/ames-housing-dataset/'<load_from_csv>
submission = pd.read_csv('.. /input/sample_sub.csv')
 submission.head()
house_data = pd.read_csv(input_path2 + 'AmesHousing.csv')
 test = pd.read_csv(input_path1 + 'test.csv')
 data_w = house_data.copy()
data_w.columns = data_w.columns.str.replace(' ', '')
 data_w.info()<concatenate>
predictions = []
for inputs, labels in testloader:
if gpu:
inputs = Variable(inputs.cuda())
 else:
inputs = Variable(inputs)
 outputs = model(inputs)
 _, predicted = torch.max(outputs.data, 1)
 for i in range(len(predicted)) :
predictions.append(int(predicted[i]))
submission['Category'] = predictions
<feature_engineering><EOS> 
submission.to_csv('Submission5.csv', index=False, encoding='utf-8' )