Kaggle competition: IESB - 2019
<SOS> metric: rmse Kaggle data source: iesb-2019<choose_model_class>
myfile = pd.read_csv(".. /input/train.csv")
 test_X = pd.read_csv(".. /input/test.csv" )
model = Net()
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 model.to(device)
 criterion = nn.CrossEntropyLoss()
optimizer = radam.RAdam(model.parameters() , lr=0.00159 )<define_variables>
test_X = test_X[['exp_vida','anos_estudo_empreendedor','idhm', 'perc_pop_econ_ativa']]
total_epoch = 50<train_model>
test_X = test_X.fillna(test_X.mean() )
n_epochs = total_epoch
train_loss_data,valid_loss_data = [],[]
valid_loss_min = np.Inf
class_correct = list(0.for i in range(10))
class_total = list(0.for i in range(10))
for epoch in range(n_epochs):
train_loss = 0.0
valid_loss = 0.0
model.train()
for data, target in trainloader:
data, target = data.to(device), target.to(device)
 optimizer.zero_grad()
output = model(data)
 loss = criterion(output, target)
 loss.backward()
optimizer.step()
train_loss += loss.item()
model.eval()
for data, target in testloader:
data, target = data.to(device), target.to(device)
 output = model(data)
 loss = criterion(output, target)
 valid_loss += loss.item()
_, pred = torch.max(output, 1)
 correct = np.squeeze(pred.eq(target.data.view_as(pred)))
 for i in range(16):
label = target.data[i]
class_correct[label] += correct[i].item()
class_total[label] += 1
train_loss = train_loss/len(trainloader.dataset)
 valid_loss = valid_loss/len(testloader.dataset)
 train_loss_data.append(train_loss)
 valid_loss_data.append(valid_loss)
 print('Epoch: {} \tTraining Loss: {:.6f} \tValidation Loss: {:.6f}'.format(
 epoch+1,
train_loss,
valid_loss 
))
print('\t\tTest Accuracy: %4d%%(%2d/%2d)' %(
 100.* np.sum(class_correct)/ np.sum(class_total),
np.sum(class_correct), np.sum(class_total)))
 if valid_loss <= valid_loss_min:
print('\t\tValidation loss decreased({:.6f} --> {:.6f} ).Saving model...'.format(
 valid_loss_min,
valid_loss))
torch.save(model.state_dict() , 'model.pt')
 valid_loss_min = valid_loss<load_from_disk>
a = myfile[['densidade_dem']]
b = myfile[['area']]
c = myfile[['codigo_mun']]
d = myfile[['ranking_igm']]
e = myfile[['comissionados_por_servidor']]
myfile['densidade_dem'] = a.apply(lambda x: x.str.replace(',',''))
myfile['area'] = b.apply(lambda x: x.str.replace(',',''))
myfile['codigo_mun'] = c.apply(lambda x: x.str.replace('ID_',''))
myfile['ranking_igm'] = d.apply(lambda x: x.str.replace('º',''))
myfile['comissionados_por_servidor'] = e.apply(lambda x: x.str.replace('%',''))
e = myfile[['comissionados_por_servidor']]
myfile['comissionados_por_servidor'] = e.apply(lambda x: x.str.replace('
myfile['densidade_dem'] = pd.to_numeric(myfile['densidade_dem'])
 myfile['area'] = pd.to_numeric(myfile['area'])
 myfile['codigo_mun'] = pd.to_numeric(myfile['codigo_mun'])
 myfile['ranking_igm'] = pd.to_numeric(myfile['ranking_igm'])
 myfile['comissionados_por_servidor'] = pd.to_numeric(myfile['comissionados_por_servidor'] )
model.load_state_dict(torch.load('model.pt'))<find_best_params>
a = myfile['regiao']
regioes = pd.get_dummies(a)
 myfile = pd.concat([myfile, regioes], axis=1 )
classes = ['0', '1', '2', '3', '4',
'5', '6', '7', '8', '9']
test_loss = 0.0
class_correct = list(0.for i in range(10))
class_total = list(0.for i in range(10))
with torch.no_grad() :
model.eval()
for data, target in testloader:
data, target = data.to(device), target.to(device)
 output = model(data)
 loss = criterion(output, target)
 test_loss += loss.item() *data.size(0)
 _, pred = torch.max(output, 1)
 correct = np.squeeze(pred.eq(target.data.view_as(pred)))
 for i in range(16):
label = target.data[i]
class_correct[label] += correct[i].item()
class_total[label] += 1
test_loss = test_loss/len(testloader.dataset)
 print('Test Loss: {:.6f}
'.format(test_loss))
for i in range(10):
if class_total[i] > 0:
print('Test Accuracy of %5s: %2d%%(%2d/%2d)' %(
 classes[i], 100 * class_correct[i] / class_total[i],
np.sum(class_correct[i]), np.sum(class_total[i])))
 else:
print('Test Accuracy of %5s: N/A(no training examples)' %(classes[i]))
print('
Test Accuracy(Overall): %2d%%(%2d/%2d)' %(
 100.* np.sum(class_correct)/ np.sum(class_total),
np.sum(class_correct), np.sum(class_total)) )<load_from_csv>
a = myfile['estado']
estado = pd.get_dummies(a)
 myfile = pd.concat([myfile, estado], axis=1 )
class DatasetSubmissionMNIST(torch.utils.data.Dataset):
def __init__(self, file_path, transform=None):
self.data = pd.read_csv(file_path)
 self.transform = transform
def __len__(self):
return len(self.data)
 def __getitem__(self, index):
image = self.data.iloc[index].values.astype(np.uint8 ).reshape(( 28, 28, 1))
if self.transform is not None:
image = self.transform(image)
 return image<load_from_csv>
a = myfile['porte']
porte = pd.get_dummies(a)
 myfile = pd.concat([myfile, porte], axis=1 )
transform = transforms.Compose([
transforms.ToPILImage() ,
transforms.ToTensor() ,
transforms.Normalize(mean=(0.5,), std=(0.5,))
])
 submissionset = DatasetSubmissionMNIST('.. /input/test.csv', transform=transform)
 submissionloader = torch.utils.data.DataLoader(submissionset, batch_size=128, shuffle=False )<find_best_params>
myfile.isnull().sum()
submission = [['ImageId', 'Label']]
with torch.no_grad() :
model.eval()
image_id = 1
for images in submissionloader:
images = images.to(device)
 log_ps = model(images)
 ps = torch.exp(log_ps)
 top_p, top_class = ps.topk(1, dim=1)
 for prediction in top_class:
submission.append([image_id, prediction.item() ])
 image_id += 1
print(len(submission)- 1 )<save_to_csv>
train_df = myfile[['nota_mat','exp_vida',
'anos_estudo_empreendedor','idhm', 'perc_pop_econ_ativa']]
train_df.dropna(inplace=True)
 train_X = train_df.drop(columns=['nota_mat'])
 train_X.head()
with open('submission.csv', 'w')as submissionFile:
writer = csv.writer(submissionFile)
 writer.writerows(submission)
 print('Submission Complete!' )<import_modules>
train_y = train_df[['nota_mat']]
train_y.head()
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename)) 
<load_from_csv>
model = Sequential()
n_cols = train_X.shape[1]
model.add(Dense(10, activation='relu', input_shape=(n_cols,)))
 model.add(Dense(10, activation='relu'))
model.add(Dense(1))
mnist_train_complete = pd.read_csv("/kaggle/input/digit-recognizer/train.csv")
 mnist_test_complete = pd.read_csv("/kaggle/input/digit-recognizer/test.csv")
 mnist_train_complete.head(5 )<data_type_conversions>
model.compile(optimizer='adam', loss='mean_squared_error' )
train_y = mnist_train_complete.iloc[:, 0].values.astype('int32')
 train_x = mnist_train_complete.iloc[:, 1:].values.astype('float32')
 test_x = mnist_test_complete.values.astype('float32')
 train_x = train_x.reshape(train_x.shape[0], 28, 28)
 test_x = test_x.reshape(test_x.shape[0], 28, 28 )<feature_engineering>
early_stopping_monitor = EarlyStopping(patience=3)
 model.fit(train_X, train_y, validation_split=0.1, epochs=30, callbacks=[early_stopping_monitor] )
train_x = train_x.astype('float32')/np.max(train_x)
 test_x = test_x.astype('float32')/np.max(test_x)
 mean = np.std(train_x)
 train_x -= mean
mean = np.std(test_x)
 test_x -= mean<split>
test_y_predictions = model.predict(test_X )
splitted_train_X, splitted_test_X, splitted_train_y, splitted_test_y = train_test_split(train_x, train_y, test_size=0.2, random_state=81)
 ohe_splitted_train_y = tf_utils.to_categorical(splitted_train_y, 10)
 ohe_splitted_test_y = tf_utils.to_categorical(splitted_test_y, 10)
 print('One-hot labels:')
 print(splitted_train_y[:10] )<choose_model_class>
test_y_predictions = pd.DataFrame(test_y_predictions )
model_sol_1 = tf.keras.models.Sequential()
model_sol_1.add(tf.keras.layers.Flatten(input_shape = splitted_train_X.shape[1:]))
model_sol_1.add(tf.keras.layers.Dense(512, activation='relu'))
model_sol_1.add(tf.keras.layers.Dropout(0.2))
model_sol_1.add(tf.keras.layers.Dense(512, activation='relu'))
model_sol_1.add(tf.keras.layers.Dropout(0.2))
model_sol_1.add(tf.keras.layers.Dense(10, activation='softmax'))
model_sol_1.summary()<choose_model_class>
test_X = pd.read_csv(".. /input/test.csv")
 b = test_X[['codigo_mun']]
b['codigo_mun'] = b.apply(lambda x: x.str.replace('ID_ID_',''))
b['codigo_mun'] =(b['codigo_mun'] ).astype(int)
 a = pd.concat([b, test_y_predictions], axis=1)
 a.columns = ['codigo_mun', 'nota_mat']
model_sol_1.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy'] )<compute_test_metric>
a.to_csv('Maria_de_Fatima.csv', index = False )