Kaggle competition: Competencia-Series-Temporales
<SOS> metric: rmse Kaggle data source: competencia-series-temporales<save_to_csv>
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
submission.to_csv("submission.csv", index=False )<save_to_csv>
train = pd.read_csv('/kaggle/input/seriestemporales-diplomado/train.txt', sep = ';', index_col=0)
 test = pd.read_csv('/kaggle/input/seriestemporales-diplomado/test.txt', sep = ';', index_col=0 )
submission.to_csv("submission.csv", index=False )<install_modules>
train = train.set_index('Fecha_Hora' )
!pip install fastai==1.0.52<import_modules>
test['Fecha_Hora'] = test['Fecha'] + ' ' + test['Hora']
from pathlib import Path
from fastai.vision import *
import torch
import random<define_variables>
test = test.drop('Fecha', axis = 1)
 test = test.drop('Hora', axis = 1)
 test.head()
PATH = Path('.. /input')

SEED = 2019
random.seed(SEED)
 torch.manual_seed(SEED)
 torch.backends.cudnn.deterministic = True
if torch.cuda.is_available() : torch.cuda.manual_seed_all(SEED)
 np.random.seed(SEED )<define_variables>
test = test[['Fecha_Hora','Poder_Reactivo_Global', 'Voltaje', 'Intensidad_Global', 'Medida_1',
'Medida_2', 'Medida_3']]
batch_size = 96<feature_engineering>
test = test.set_index('Fecha_Hora' )
tfms = get_transforms(flip_vert=True )<load_from_csv>
result = [train, test]
dataset = pd.concat(result )
src =(ImageList.from_csv(csv_name='train.csv', path=PATH, folder='train/train')
.split_by_rand_pct(0.1)
.label_from_df(cols='has_cactus'))<categorify>
dataset = dataset.replace('?', np.nan )
data =(src.transform(tfms,size=128)
.databunch(bs=batch_size)
.normalize(imagenet_stats))<define_variables>
import seaborn as sns
data.show_batch(rows=3, figsize=(9, 9))<find_best_params>
dataset.isnull().sum()
data.classes, data.c<choose_model_class>
set_test = dataset.iloc[-8760:,:]
learn = cnn_learner(data, models.densenet161, metrics=accuracy, path='./' )<train_model>
set_train = dataset.iloc[:-8760,:]
lr = 3e-2
learn.fit_one_cycle(4,slice(lr))<categorify>
set_train
data =(src.transform(tfms, size=128)
.add_test_folder('test/test')
.databunch() 
.normalize(imagenet_stats))<prepare_output>
set_test
learn.data = data<predict_on_test>
droping_list_all=[]
for j in range(0,7):
if not dataset.iloc[:, j].notnull().all() :
droping_list_all.append(j)
 droping_list_all
probs, _ = learn.get_preds(ds_type=DatasetType.Test )<prepare_x_and_y>
set_train = set_train.apply(pd.to_numeric )
ilst = data.test_ds.x<define_variables>
set_train.iloc[:,0]=set_train.iloc[:,0].fillna(set_train.iloc[:,0].mean() )
fnames = [item.name for item in ilst.items]; fnames[:10]<prepare_output>
set_train.iloc[:,1]=set_train.iloc[:,1].fillna(set_train.iloc[:,1].mean() )
test_df = pd.DataFrame({'id': fnames, 'has_cactus': probs.numpy() [:,0]}); test_df.head()<save_to_csv>
set_train.isnull().sum()
test_df.to_csv('submission.csv', index=None )<set_options>
set_train = set_train[['Intensidad_Global', 'Medida_1', 'Medida_2', 'Medida_3', 'Poder_Reactivo_Global', 'Voltaje',
'Poder_Activo_Global']]
set_train
%matplotlib inline<load_from_csv>
set_test = set_test[['Intensidad_Global', 'Medida_1', 'Medida_2', 'Medida_3', 'Poder_Reactivo_Global', 'Voltaje',
'Poder_Activo_Global']]
set_test
labels = pd.read_csv('.. /input/train.csv')
 sub = pd.read_csv('.. /input/sample_submission.csv')
 train_path = '.. /input/train/train/'
test_path = '.. /input/test/test/' 
<define_variables>
values = set_train.values
scaler = MinMaxScaler(feature_range=(0, 1))
scaled = scaler.fit_transform(values )
print('Num train samples:{0}'.format(len(os.listdir(train_path))))
print('Num test samples:{0}'.format(len(os.listdir(test_path))))<count_values>
values = scaled
n_train_time_start = 1920000
n_train_time = 2000000
train = values[n_train_time_start:n_train_time, :]
test = values[n_train_time:, :]
train_X, train_y = train[:, :-1], train[:, -1]
test_X, test_y = test[:, :-1], test[:, -1]
train_X = train_X.reshape(( train_X.shape[0], 1, train_X.shape[1]))
test_X = test_X.reshape(( test_X.shape[0], 1, test_X.shape[1]))
print(train_X.shape, train_y.shape, test_X.shape, test_y.shape )
labels['has_cactus'].value_counts()<import_modules>
def root_mean_squared_error(y_true, y_pred):
return K.sqrt(K.mean(K.square(y_pred - y_true)) )
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.data import TensorDataset, DataLoader, Dataset
import torchvision
import torchvision.transforms as transforms
from sklearn.model_selection import train_test_split<init_hyperparams>
num_epochs = 25
num_classes = 2
batch_size = 128
learning_rate = 0.002
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu' )<split>
yhat = model.predict(test_X)
 test_X = test_X.reshape(( test_X.shape[0], 6))
inv_yhat = np.concatenate(( yhat, test_X[:, -6:]), axis=1)
 inv_yhat = scaler.inverse_transform(inv_yhat)
 inv_yhat = inv_yhat[:,0]
test_y = test_y.reshape(( len(test_y), 1))
inv_y = np.concatenate(( test_y, test_X[:, -6:]), axis=1)
 inv_y = scaler.inverse_transform(inv_y)
 inv_y = inv_y[:,0]
rmse = np.sqrt(mean_squared_error(inv_y, inv_yhat))
print('Test RMSE: %.3f' % rmse )
train, val = train_test_split(labels, stratify=labels.has_cactus, test_size=0.1)
 train.shape, val.shape<count_values>
set_test = set_test.apply(pd.to_numeric )
train['has_cactus'].value_counts()<count_values>
set_test.isnull().sum()
val['has_cactus'].value_counts()<categorify>
set_test_to_predict = set_test[['Intensidad_Global', 'Medida_1', 'Medida_2', 'Medida_3', 'Poder_Reactivo_Global', 'Voltaje']]
class MyDataset(Dataset):
def __init__(self, df_data, data_dir = './', transform=None):
super().__init__()
self.df = df_data.values
self.data_dir = data_dir
self.transform = transform
def __len__(self):
return len(self.df)
 def __getitem__(self, index):
img_name,label = self.df[index]
img_path = os.path.join(self.data_dir, img_name)
 image = cv2.imread(img_path)
 if self.transform is not None:
image = self.transform(image)
 return image, label<categorify>
set_test_to_predict
trans_train = transforms.Compose([transforms.ToPILImage() ,
transforms.Pad(32, padding_mode='reflect'),
transforms.ToTensor() ,
transforms.Normalize(mean=[0.5, 0.5, 0.5],std=[0.5, 0.5, 0.5])])
 trans_valid = transforms.Compose([transforms.ToPILImage() ,
transforms.Pad(32, padding_mode='reflect'),
transforms.ToTensor() ,
transforms.Normalize(mean=[0.5, 0.5, 0.5],std=[0.5, 0.5, 0.5])])
 dataset_train = MyDataset(df_data=train, data_dir=train_path, transform=trans_train)
 dataset_valid = MyDataset(df_data=val, data_dir=train_path, transform=trans_valid)
 loader_train = DataLoader(dataset = dataset_train, batch_size=batch_size, shuffle=True, num_workers=0)
 loader_valid = DataLoader(dataset = dataset_valid, batch_size=batch_size//2, shuffle=False, num_workers=0 )<choose_model_class>
values = set_test_to_predict.values
scaler_test = MinMaxScaler(feature_range=(0, 1))
scaled_test = scaler_test.fit_transform(values )
class SimpleCNN(nn.Module):
def __init__(self):
super(SimpleCNN, self ).__init__()
self.conv1 = nn.Conv2d(in_channels=3, out_channels=32, kernel_size=3, padding=2)
 self.conv2 = nn.Conv2d(in_channels=32, out_channels=64, kernel_size=3, padding=2)
 self.conv3 = nn.Conv2d(in_channels=64, out_channels=128, kernel_size=3, padding=2)
 self.conv4 = nn.Conv2d(in_channels=128, out_channels=256, kernel_size=3, padding=2)
 self.conv5 = nn.Conv2d(in_channels=256, out_channels=512, kernel_size=3, padding=2)
 self.bn1 = nn.BatchNorm2d(32)
 self.bn2 = nn.BatchNorm2d(64)
 self.bn3 = nn.BatchNorm2d(128)
 self.bn4 = nn.BatchNorm2d(256)
 self.bn5 = nn.BatchNorm2d(512)
 self.pool = nn.MaxPool2d(kernel_size=2, stride=2)
 self.avg = nn.AvgPool2d(4)
 self.fc = nn.Linear(512 * 1 * 1, 2)
 def forward(self, x):
x = self.pool(F.leaky_relu(self.bn1(self.conv1(x))))
x = self.pool(F.leaky_relu(self.bn2(self.conv2(x))))
x = self.pool(F.leaky_relu(self.bn3(self.conv3(x))))
x = self.pool(F.leaky_relu(self.bn4(self.conv4(x))))
x = self.pool(F.leaky_relu(self.bn5(self.conv5(x))))
x = self.avg(x)
 x = x.view(-1, 512 * 1 * 1)
 x = self.fc(x)
 return x<choose_model_class>
to_predict = scaled_test
to_predict = to_predict.reshape(( to_predict.shape[0], 1, to_predict.shape[1]))
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adamax(model.parameters() , lr=learning_rate )<train_model>
ypredict = model.predict(to_predict )
total_step = len(loader_train)
 for epoch in range(num_epochs):
for i,(images, labels)in enumerate(loader_train):
images = images.to(device)
 labels = labels.to(device)
 outputs = model(images)
 loss = criterion(outputs, labels)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
if(i+1)% 100 == 0:
print('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}' 
.format(epoch+1, num_epochs, i+1, total_step, loss.item()))<train_model>
to_predict = to_predict.reshape(( to_predict.shape[0], 6))
inv_ypredict = np.concatenate(( to_predict[:, :], ypredict), axis=1 )
model.eval()
with torch.no_grad() :
correct = 0
total = 0
for images, labels in loader_valid:
images = images.to(device)
 labels = labels.to(device)
 outputs = model(images)
 _, predicted = torch.max(outputs.data, 1)
 total += labels.size(0)
 correct +=(predicted == labels ).sum().item()
print('Test Accuracy of the model on the 1750 validation images: {} %'.format(100 * correct / total))
torch.save(model.state_dict() , 'model.ckpt' )<load_pretrained>
inv_ypredict = scaler.inverse_transform(inv_ypredict)
 ypredict_final = inv_ypredict[:,-1]
dataset_valid = MyDataset(df_data=sub, data_dir=test_path, transform=trans_valid)
 loader_test = DataLoader(dataset = dataset_valid, batch_size=32, shuffle=False, num_workers=0 )<save_to_csv>
prediction = pd.DataFrame(ypredict_final )
model.eval()
preds = []
for batch_i,(data, target)in enumerate(loader_test):
data, target = data.cuda() , target.cuda()
output = model(data)
 pr = output[:,1].detach().cpu().numpy()
for i in pr:
preds.append(i)
 sub['has_cactus'] = preds
sub.to_csv('sub.csv', index=False )<set_options>
set_test['Poder_Activo_Global'] = ypredict_final
%reload_ext autoreload
%autoreload 2
%matplotlib inline<import_modules>
set_test
from fastai import *
from fastai.vision import *<define_variables>
enviar = set_test[['Poder_Activo_Global']]
PATH = ".. /input/"
sz=32
bs=512<load_from_csv>
enviar['Fecha_Hora'] = enviar.index
tfms = get_transforms(flip_vert=True, max_rotate=90.)
data = ImageDataBunch.from_csv(PATH, ds_tfms=tfms,
folder="train/train", csv_labels='train.csv', test="test/test",
valid_pct=0.1, fn_col=0, label_col=1 ).normalize(imagenet_stats )<choose_model_class>
enviar = enviar.reset_index(drop=True)
 enviar = enviar[['Fecha_Hora', 'Poder_Activo_Global']]
enviar
learn = create_cnn(data, models.resnet50, metrics=accuracy, path='.. /kaggle/working', model_dir='.. /kaggle/working/model',callback_fns=ShowGraph )<find_best_params>
enviar.to_csv('Output.csv',index=False )