Kaggle competition: TMDB Box Office Prediction
<SOS> metric: RMSLE Kaggle data source: tmdb-box-office-prediction<load_pretrained>
pd.set_option('max_columns', None)
 %matplotlib inline
plt.style.use('ggplot')
 stop = set(stopwords.words('english'))
warnings.filterwarnings("ignore" )
if ComputeLB:
with zipfile.ZipFile(".. /working/images.zip","r")as z:
z.extractall(".. /tmp/images2/")
 m2 = []; s2 =[]; f2 = []
user_images_unzipped_path = '.. /tmp/images2/'
images_path = [user_images_unzipped_path,'.. /input/generative-dog-images/all-dogs/all-dogs/']
public_path = '.. /input/dog-face-generation-competition-kid-metric-input/classify_image_graph_def.pb'
fid_epsilon = 10e-15
fid_value_public, distance_public, m2, s2, f2 = calculate_kid_given_paths(images_path, 'Inception', public_path, mm=m2, ss=s2, ff=f2)
 distance_public = distance_thresholding(distance_public, model_params['Inception']['cosine_distance_eps'])
 print("FID_public: ", fid_value_public, "distance_public: ", distance_public, "multiplied_public: ",
fid_value_public /(distance_public + fid_epsilon))
! rm -r.. /tmp<define_variables>
train=pd.read_csv('.. /input/train.csv',index_col=0)
 test=pd.read_csv('.. /input/test.csv',index_col=0 )
batch_size = 32<define_variables>
!pip install --upgrade pip
!pip install imdbpy
ia=IMDb()
def fill_col(df,col,key,fill_missing_type,nested_arr):
indexes=df.index[df[col]!=df[col]].tolist()
for index in indexes:
imdb_id=df.loc[index]['imdb_id']
movie=ia.get_movie(imdb_id[2:])
 if key in movie:
if(nested_arr):
arr=[]
for el in movie[key]:
arr.append(str(el))
df.at[index, col] = ','.join(arr)
 else:
df.at[index, col] = ','.join(movie[key])
 else:
if(fill_missing_type=='mean'):
df.at[index, col] = df[col].mean()
elif(fill_missing_type=='others'):
df.at[index, col] = 'others'
elif(fill_missing_type=='None'):
df.at[index, col] = None
fill_col(train,'runtime','runtimes','None',False)
 fill_col(test,'runtime','runtimes','None',False )
start = time()<set_options>
test['revenue']=-99
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()<load_pretrained>
train['be_is_na']=train['belongs_to_collection'].apply(lambda x :1 if x is np.nan else 0)
 test['be_is_na']=test['belongs_to_collection'].apply(lambda x :1 if x is np.nan else 0)
 train=train.drop('belongs_to_collection',axis=1)
 test=test.drop('belongs_to_collection',axis=1 )
class DataGenerator(Dataset):
def __init__(self, directory, transform=None, n_samples=np.inf):
self.directory = directory
self.transform = transform
self.n_samples = n_samples
self.samples = self._load_subfolders_images(directory)
 if len(self.samples)== 0:
raise RuntimeError("Found 0 files in subfolders of: {}".format(directory))
def _load_subfolders_images(self, root):
IMG_EXTENSIONS =(
 '.jpg', '.jpeg', '.png', '.ppm', '.bmp', '.pgm', '.tif', '.tiff', '.webp')
 def is_valid_file(x):
return torchvision.datasets.folder.has_file_allowed_extension(x, IMG_EXTENSIONS)
 required_transforms = torchvision.transforms.Compose([
torchvision.transforms.Resize(64),
torchvision.transforms.CenterCrop(64),
])
 imgs = []
paths = []
for root, _, fnames in sorted(os.walk(root)) :
for fname in sorted(fnames)[:min(self.n_samples, 999999999999999)]:
path = os.path.join(root, fname)
 paths.append(path)
 for path in paths:
if is_valid_file(path):
img = dset.folder.default_loader(path)
 annotation_basename = os.path.splitext(os.path.basename(path)) [0]
annotation_dirname = next(
 dirname for dirname in os.listdir('.. /input/annotation/Annotation/')if
dirname.startswith(annotation_basename.split('_')[0]))
annotation_filename = os.path.join('.. /input/annotation/Annotation/',
annotation_dirname, annotation_basename)
 tree = ET.parse(annotation_filename)
 root = tree.getroot()
objects = root.findall('object')
 for o in objects:
bndbox = o.find('bndbox')
 xmin = int(bndbox.find('xmin' ).text)
 ymin = int(bndbox.find('ymin' ).text)
 xmax = int(bndbox.find('xmax' ).text)
 ymax = int(bndbox.find('ymax' ).text)
 w = np.min(( xmax - xmin, ymax - ymin))
bbox =(xmin, ymin, xmin+w, ymin+w)
 object_img = required_transforms(img.crop(bbox))
imgs.append(object_img)
 return imgs
def __getitem__(self, index):
sample = self.samples[index]
if self.transform is not None:
sample = self.transform(sample)
 return np.asarray(sample)
 def __len__(self):
return len(self.samples)
<categorify>
train.loc[train.index == 16,'revenue'] = 192864
train.loc[train.index == 90,'budget'] = 30000000
train.loc[train.index== 118,'budget'] = 60000000
train.loc[train.index== 149,'budget'] = 18000000
train.loc[train.index== 313,'revenue'] = 12000000
train.loc[train.index == 451,'revenue'] = 12000000
train.loc[train.index == 464,'budget'] = 20000000
train.loc[train.index == 470,'budget'] = 13000000
train.loc[train.index== 513,'budget'] = 930000
train.loc[train.index == 797,'budget'] = 8000000
train.loc[train.index == 819,'budget'] = 90000000
train.loc[train.index == 850,'budget'] = 90000000
train.loc[train.index == 1007,'budget'] = 2
train.loc[train.index== 1112,'budget'] = 7500000
train.loc[train.index == 1131,'budget'] = 4300000
train.loc[train.index == 1359,'budget'] = 10000000
train.loc[train.index == 1542,'budget'] = 1
train.loc[train.index == 1570,'budget'] = 15800000
train.loc[train.index== 1571,'budget'] = 4000000
train.loc[train.index == 1714,'budget'] = 46000000
train.loc[train.index == 1721,'budget'] = 17500000
train.loc[train.index== 1865,'revenue'] = 25000000
train.loc[train.index == 1885,'budget'] = 12
train.loc[train.index == 2091,'budget'] = 10
train.loc[train.index == 2268,'budget'] = 17500000
train.loc[train.index == 2491,'budget'] = 6
train.loc[train.index == 2602,'budget'] = 31000000
train.loc[train.index == 2612,'budget'] = 15000000
train.loc[train.index == 2696,'budget'] = 10000000
train.loc[train.index == 2801,'budget'] = 10000000
train.loc[train.index == 335,'budget'] = 2
train.loc[train.index == 348,'budget'] = 12
train.loc[train.index == 470,'budget'] = 13000000
train.loc[train.index == 513,'budget'] = 1100000
train.loc[train.index == 640,'budget'] = 6
train.loc[train.index == 696,'budget'] = 1
train.loc[train.index == 797,'budget'] = 8000000
train.loc[train.index == 850,'budget'] = 1500000
train.loc[train.index == 1199,'budget'] = 5
train.loc[train.index == 1282,'budget'] = 9
train.loc[train.index== 1347,'budget'] = 1
train.loc[train.index== 1755,'budget'] = 2
train.loc[train.index == 1801,'budget'] = 5
train.loc[train.index == 1918,'budget'] = 592
train.loc[train.index == 2033,'budget'] = 4
train.loc[train.index == 2118,'budget'] = 344
train.loc[train.index == 2252,'budget'] = 130
train.loc[train.index == 2256,'budget'] = 1
train.loc[train.index == 2696,'budget'] = 10000000
%%time
database = '.. /input/all-dogs/all-dogs/'
transform = transforms.Compose([transforms.RandomHorizontalFlip(p=0.3),
transforms.ToTensor() ,
transforms.Normalize(( 0.5, 0.5, 0.5),(0.5, 0.5, 0.5)) ])
 train_data = DataGenerator(database, transform=transform,n_samples=25000)
 train_loader = torch.utils.data.DataLoader(train_data, shuffle=True,batch_size=batch_size, num_workers = 4)
<normalization>
test.loc[test.index== 3033,'budget'] = 250
test.loc[test.index== 3051,'budget'] = 50
test.loc[test.index == 3084,'budget'] = 337
test.loc[test.index == 3224,'budget'] = 4
test.loc[test.index == 3594,'budget'] = 25
test.loc[test.index == 3619,'budget'] = 500
test.loc[test.index == 3831,'budget'] = 3
test.loc[test.index== 3935,'budget'] = 500
test.loc[test.index == 4049,'budget'] = 995946
test.loc[test.index== 4424,'budget'] = 3
test.loc[test.index == 4460,'budget'] = 8
test.loc[test.index == 4555,'budget'] = 1200000
test.loc[test.index== 4624,'budget'] = 30
test.loc[test.index== 4645,'budget'] = 500
test.loc[test.index == 4709,'budget'] = 450
test.loc[test.index == 4839,'budget'] = 7
test.loc[test.index== 3125,'budget'] = 25
test.loc[test.index== 3142,'budget'] = 1
test.loc[test.index == 3201,'budget'] = 450
test.loc[test.index == 3222,'budget'] = 6
test.loc[test.index== 3545,'budget'] = 38
test.loc[test.index == 3670,'budget'] = 18
test.loc[test.index == 3792,'budget'] = 19
test.loc[test.index == 3881,'budget'] = 7
test.loc[test.index == 3969,'budget'] = 400
test.loc[test.index == 4196,'budget'] = 6
test.loc[test.index == 4221,'budget'] = 11
test.loc[test.index == 4222,'budget'] = 500
test.loc[test.index== 4285,'budget'] = 11
test.loc[test.index == 4319,'budget'] = 1
test.loc[test.index == 4639,'budget'] = 10
test.loc[test.index == 4719,'budget'] = 45
test.loc[test.index == 4822,'budget'] = 22
test.loc[test.index == 4829,'budget'] = 20
test.loc[test.index== 4969,'budget'] = 20
test.loc[test.index== 5021,'budget'] = 40
test.loc[test.index== 5035,'budget'] = 1
test.loc[test.index== 5063,'budget'] = 14
test.loc[test.index == 5119,'budget'] = 2
test.loc[test.index== 5214,'budget'] = 30
test.loc[test.index== 5221,'budget'] = 50
test.loc[test.index== 4903,'budget'] = 15
test.loc[test.index == 4983,'budget'] = 3
test.loc[test.index == 5102,'budget'] = 28
test.loc[test.index== 5217,'budget'] = 75
test.loc[test.index == 5224,'budget'] = 3
test.loc[test.index== 5469,'budget'] = 20
test.loc[test.index == 5840,'budget'] = 1
test.loc[test.index == 5960,'budget'] = 30
test.loc[test.index == 6506,'budget'] = 11
test.loc[test.index== 6553,'budget'] = 280
test.loc[test.index == 6561,'budget'] = 7
test.loc[test.index== 6582,'budget'] = 218
test.loc[test.index == 6638,'budget'] = 5
test.loc[test.index== 6749,'budget'] = 8
test.loc[test.index==6759,'budget'] = 50
test.loc[test.index == 6856,'budget'] = 10
test.loc[test.index== 6858,'budget'] = 100
test.loc[test.index == 6876,'budget'] = 250
test.loc[test.index == 6972,'budget'] = 1
test.loc[test.index== 7079,'budget'] = 8000000
test.loc[test.index == 7150,'budget'] = 118
test.loc[test.index == 6506,'budget'] = 118
test.loc[test.index == 7225,'budget'] = 6
test.loc[test.index == 7231,'budget'] = 85
test.loc[test.index == 5222,'budget'] = 5
test.loc[test.index == 5322,'budget'] = 90
test.loc[test.index == 5350,'budget'] = 70
test.loc[test.index == 5378,'budget'] = 10
test.loc[test.index== 5545,'budget'] = 80
test.loc[test.index == 5810,'budget'] = 8
test.loc[test.index== 5926,'budget'] = 300
test.loc[test.index== 5927,'budget'] = 4
test.loc[test.index== 5986,'budget'] = 1
test.loc[test.index == 6053,'budget'] = 20
test.loc[test.index== 6104,'budget'] = 1
test.loc[test.index == 6130,'budget'] = 30
test.loc[test.index == 6301,'budget'] = 150
test.loc[test.index == 6276,'budget'] = 100
test.loc[test.index == 6473,'budget'] = 100
test.loc[test.index== 6842,'budget'] = 30
class PixelwiseNorm(nn.Module):
def __init__(self):
super(PixelwiseNorm, self ).__init__()
def forward(self, x, alpha=1e-8):

 y = x.pow(2.).mean(dim=1, keepdim=True ).add(alpha ).sqrt()
y = x / y
return y<categorify>
new_data=pd.concat([train,test],axis=0)
 new_data.head()
class MinibatchStdDev(th.nn.Module):

 def __init__(self):

 super(MinibatchStdDev, self ).__init__()
def forward(self, x, alpha=1e-8):

 batch_size, _, height, width = x.shape
y = x - x.mean(dim=0, keepdim=True)
 y = th.sqrt(y.pow(2.).mean(dim=0, keepdim=False)+ alpha)
 y = y.mean().view(1, 1, 1, 1)
 y = y.repeat(batch_size,1, height, width)
 y = th.cat([x, y], 1)
 return y 
<define_search_model>
new_data.isnull().sum()
class Generator(nn.Module):
def __init__(self, nz, nfeats, nchannels):
super(Generator, self ).__init__()
self.conv1 = spectral_norm(nn.ConvTranspose2d(nz, nfeats * 8, 4, 1, 0, bias=False))
self.conv2 = spectral_norm(nn.ConvTranspose2d(nfeats * 8, nfeats * 8, 4, 2, 1, bias=False))
self.conv3 = spectral_norm(nn.ConvTranspose2d(nfeats * 8, nfeats * 4, 4, 2, 1, bias=False))
self.conv4 = spectral_norm(nn.ConvTranspose2d(nfeats * 4, nfeats * 2, 4, 2, 1, bias=False))
self.conv5 = spectral_norm(nn.ConvTranspose2d(nfeats * 2, nfeats, 4, 2, 1, bias=False))
self.conv6 = spectral_norm(nn.ConvTranspose2d(nfeats, nchannels, 3, 1, 1, bias=False))
self.pixnorm = PixelwiseNorm()
def forward(self, x):
x = F.leaky_relu(self.conv1(x))
x = F.leaky_relu(self.conv2(x))
x = self.pixnorm(x)
 x = F.leaky_relu(self.conv3(x))
x = self.pixnorm(x)
 x = F.leaky_relu(self.conv4(x))
x = self.pixnorm(x)
 x = F.leaky_relu(self.conv5(x))
x = self.pixnorm(x)
 x = torch.tanh(self.conv6(x))
return x
class Discriminator(nn.Module):
def __init__(self, nchannels, nfeats):
super(Discriminator, self ).__init__()
self.conv1 = nn.Conv2d(nchannels, nfeats, 4, 2, 1, bias=False)
 self.conv2 = spectral_norm(nn.Conv2d(nfeats, nfeats * 2, 4, 2, 1, bias=False))
self.bn2 = nn.BatchNorm2d(nfeats * 2)
 self.conv3 = spectral_norm(nn.Conv2d(nfeats * 2, nfeats * 4, 4, 2, 1, bias=False))
self.bn3 = nn.BatchNorm2d(nfeats * 4)
 self.conv4 = spectral_norm(nn.Conv2d(nfeats * 4, nfeats * 8, 4, 2, 1, bias=False))
self.bn4 = nn.MaxPool2d(2)
 self.batch_discriminator = MinibatchStdDev()
self.pixnorm = PixelwiseNorm()
self.conv5 = spectral_norm(nn.Conv2d(nfeats * 8 +1, 1, 2, 1, 0, bias=False))
def forward(self, x):
x = F.leaky_relu(self.conv1(x), 0.2)
 x = F.leaky_relu(self.bn2(self.conv2(x)) , 0.2)
 x = F.leaky_relu(self.bn3(self.conv3(x)) , 0.2)
 x = F.leaky_relu(self.bn4(self.conv4(x)) , 0.2)
 x = self.batch_discriminator(x)
 x= self.conv5(x)
 return x.view(-1, 1)
<choose_model_class>
new_data[new_data['release_date'].isnull() ]
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 lr = 0.0003
lr_d = 0.0001
beta1 = 0.5
epochs = 500
netG = Generator(100, 32, 3 ).to(device)
 netD = Discriminator(3, 48 ).to(device)
 criterion = nn.BCELoss()
optimizerD = optim.RMSprop(netD.parameters() , lr=lr, alpha=0.99, eps=1e-08)
 optimizerG = optim.RMSprop(netG.parameters() , lr=lr_d, alpha=0.99, eps=1e-08)
 lr_schedulerG = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizerG,
T_0=epochs//200, eta_min=0.00005)
 lr_schedulerD = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizerD,
T_0=epochs//200, eta_min=0.00005)
 nz = 100
fixed_noise = torch.randn(25, nz, 1, 1, device=device)
 real_label = 0.7
fake_label = 0.0 
<find_best_params>
new_data['release_date']=new_data['release_date'].fillna('3/20/01' )
gpu = 0
LAMBDA = 10
use_cuda = torch.cuda.is_available()
if use_cuda:
gpu = 0
def calc_gradient_penalty(netD, real_data, fake_data):
BATCH_SIZE = real_data.size() [0]
alpha = torch.rand(BATCH_SIZE, 1)
 alpha = alpha.expand(BATCH_SIZE, real_data.nelement() //BATCH_SIZE ).contiguous().view(BATCH_SIZE, 3, 64, 64)
 alpha = alpha.cuda(gpu)if use_cuda else alpha
interpolates = alpha * real_data +(( 1 - alpha)* fake_data)
 interpolates = interpolates.cuda(gpu)
 interpolates = autograd.Variable(interpolates, requires_grad=True)
 disc_interpolates = netD(interpolates)
 gradients = autograd.grad(outputs=disc_interpolates, inputs=interpolates,
grad_outputs=torch.ones(disc_interpolates.size() ).cuda(gpu)if True else torch.ones(
 disc_interpolates.size()),
create_graph=True, retain_graph=True, only_inputs=True)[0]
gradients = gradients.view(gradients.size(0), -1)
 gradient_penalty =(( gradients.norm(2, dim=1)- 1)** 2 ).mean() * LAMBDA
return gradient_penalty<train_model>
new_data['release_year']=pd.to_datetime(new_data['release_date'] ).dt.year
new_data['release_month']=pd.to_datetime(new_data['release_date'] ).dt.month
new_data['release_day']=pd.to_datetime(new_data['release_date'] ).dt.day
new_data['release_dayofweek'] = pd.to_datetime(new_data['release_date'] ).dt.dayofweek
new_data['release_quarter'] = pd.to_datetime(new_data['release_date'] ).dt.quarter
gpu = 0
one = torch.FloatTensor([1])
 mone = one * -1
one = one.cuda(gpu)
 mone = mone.cuda(gpu)
 step = 0
for epoch in range(epochs):
for ii,(real_images)in tqdm(enumerate(train_loader), total=len(train_loader)) :
end = time()
if(end -start)> 25000 :
break
netD.zero_grad()
real_images = real_images.to(device)
 batch_size = real_images.size(0)
 labels = torch.full(( batch_size, 1), real_label, device=device)+ np.random.uniform(-0.1, 0.1)
 D_real = netD(real_images)
 D_real_mean = D_real.mean()
D_real_mean.backward(mone)
 D_x = D_real_mean.item()
noise = torch.randn(batch_size, nz, 1, 1, device=device)
 fake = netG(noise)
 labels.fill_(fake_label)+ np.random.uniform(0, 0.2)
 D_fake = netD(fake.detach())
 D_fake_mean = D_fake.mean()
D_fake_mean.backward(one)
 D_G_z1 = D_fake_mean.item()
errD = D_real_mean + D_fake_mean
gradient_penalty = calc_gradient_penalty(netD, real_images.data, fake.data)
 gradient_penalty.backward()
D_cost = D_fake - D_real + gradient_penalty
Wasserstein_D = D_real - D_fake
optimizerD.step()
netG.zero_grad()
labels.fill_(real_label)
 G = netD(fake)
 G_mean = G.mean()
G_mean.backward(mone)
 G_cost = -G_mean
D_G_z2 = G_mean.item()
optimizerG.step()
if step % 500 == 0:
print('[%d/%d][%d/%d] Loss_D: %.4f Loss_G: %.4f D(x): %.4f D(G(z)) : %.4f / %.4f'
%(epoch + 1, epochs, ii, len(train_loader),
errD.item() , G_mean.item() , D_x, D_G_z1, D_G_z2))
valid_image = netG(fixed_noise)
 step += 1
lr_schedulerG.step(epoch)
 lr_schedulerD.step(epoch)
 if epoch % 25 == 0:
show_generated_img() 
<categorify>
new_data['release_year'].loc[new_data['release_year']>=2018]-=100
def truncated_normal(size, threshold=1):
values = truncnorm.rvs(-threshold, threshold, size=size)
 return values
if not os.path.exists('.. /output_images'):
os.mkdir('.. /output_images')
 im_batch_size = 100
n_images=10000
for i_batch in range(0, n_images, im_batch_size):
z = truncated_normal(( im_batch_size, 100, 1, 1), threshold=1)
 gen_z = torch.from_numpy(z ).float().to(device)
 gen_images = netG(gen_z)
 images = gen_images.to("cpu" ).clone().detach()
images = images.numpy().transpose(0, 2, 3, 1)
 for i_image in range(gen_images.size(0)) :
save_image(( gen_images[i_image, :, :, :] +1.0)/2.0, os.path.join('.. /output_images', f'image_{i_batch+i_image:05d}.png'))
shutil.make_archive('images', 'zip', '.. /output_images' )<import_modules>
new_data=new_data.drop('release_date',axis=1 )
import pandas as pd
import numpy as np
import random
from sklearn.tree import DecisionTreeClassifier
from sklearn.linear_model import LogisticRegression,RidgeClassifier
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt<load_from_csv>
weiji=[1929,1930,1930,1931,1932,1933,1934,1935,1936,1937,1938,1939,1940,1941,1942,1943,1944,1945,1950,1951,1952,1953,1961,1962,1963,1964,1965,1966,1967,1968,1969,1971,1973,1974,1975,1980,1981,1982,1983,1984,1985,1986,1987,1988,1989,
1990,1991,1992,1993,1994,1995,1996,1997,1998,1999,2007,2008,2009,2010,2011]
prob_auc_df = pd.read_csv(".. /input/300-probed-aucs-from-dont-overfit-ii/probed_aucs.csv")
 prob_auc_df.loc[:,"variable"]=prob_auc_df.loc[:,"variable"].transform(lambda x: x[1:] ).astype(int)
 prob_auc_df<sort_values>
for i in weiji:
new_data['is_'+str(i)]=new_data['release_year'].apply(lambda x:1 if x==i else 0 )
prob_auc_df.sort_values("variable",inplace=True)
 prob_auc_df.reset_index(drop=True,inplace=True )<feature_engineering>
new_data['homepage_fact']=new_data['homepage'].apply(lambda x: 0 if x is np.nan else 1 )
prob_auc_df.loc[:,"public_auc"] = prob_auc_df.loc[:,"public_auc"]-0.5
prob_auc_df.loc[:,"public_auc"]=prob_auc_df.loc[:,"public_auc"].transform(lambda x: x if np.abs(x)>0.04 else 0)
 prob_auc_df<compute_test_metric>
new_data['homepage_end']=new_data[new_data['homepage'].notna() ]['homepage'].str.findall(r'\.( [a-z]+ )(?:\/|$)' ).apply(lambda x:x[0])
 new_data['homepage_end'].head()
score = np.array(prob_auc_df.loc[:,"public_auc"] )<load_from_csv>
new_data['homepage_end']=new_data['homepage_end'].fillna('unknow' )
train_df = pd.read_csv(".. /input/dont-overfit-ii/train.csv" ).drop('id', axis=1)
 train_df.head()<load_from_csv>
page=pd.get_dummies(new_data['homepage_end'])
 page.head()
test_df = pd.read_csv('.. /input/dont-overfit-ii/test.csv' ).drop('id', axis = 1)
 test_df.head()<feature_engineering>
new_data=new_data.drop('homepage',axis=1 )
prob_df = test_df.copy()
prob_df.loc[:,"target"]= expit(np.dot(np.array(prob_df),score)) >0.5<set_options>
new_data=new_data.drop('poster_path',axis=1 )
plt.figure(figsize=(12, 12))
sns.violinplot(data=test_df.dropna() ["0"],orient="h" )<compute_train_metric>
new_data['len_overview']=new_data['overview'].fillna('NAN' ).apply(lambda x:len(x))
new_data['len_overview'][new_data['len_overview']==0]=1
new_data['len_overview_budget']=new_data['budget']/(new_data['len_overview']+1 )
for col in test_df.columns:
z,p= scipy.stats.normaltest(test_df[col])
 print(p<0.005,z,p )<sort_values>
len_rew_sort=new_data['len_overview'].sort_values(ascending=True)
 len_rew_sort.head()
x_corr = test_df.corr().sort_values("0" ).sort_values("0",axis=1)
 x_corr.head()<choose_model_class>
length=len(new_data['len_overview'])
 m=0.1
n=0.1
arr_len_ove=[]
for i in range(1,11):
arr_len_ove.append(round(length*m))
m+=n
arr_len_ove
kbest = TSNE(3)
<prepare_x_and_y>
for i in range(10):
qu=qu_arr[i]
if i==0:
new_data['len_overview'].loc[(new_data['len_overview']<len_rew_sort.iloc[arr_len_ove[i]-1])]=qu
else:
new_data['len_overview'].loc[(new_data['len_overview']<len_rew_sort.iloc[arr_len_ove[i]-1])&(new_data['len_overview'] >qu_arr[i-1])]=qu
print(i,qu )
robust = RobustScaler().fit(np.concatenate(( train_df.drop('target', axis=1), test_df), axis=0))
y = train_df['target']
X = SimpleImputer(strategy='mean' ).fit_transform(train_df.drop('target', axis=1)) 
<split>
len_ove_agg=new_data.groupby('len_overview' ).revenue.aggregate(['min','max','std'] )
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42,train_size=0.8)
<compute_test_metric>
new_data['geres_name']=new_data['genres'].str.findall(r''name'\s?:\s?'(\w+)'')
 new_data['geres_name'].head()
logreg = GaussianNB()
logreg.fit(X_train, y_train)
 test_score = logreg.score(X_test, y_test)
 test_score<split>
new_data=new_data.drop('genres',axis=1 )
best_score = 0
best_std = 0
for penalty in ['l1']:
for C in [2.**(i/2)for i in range(-9*2,6*2)]:
score_ = []
for train_index, val_index in StratifiedShuffleSplit(n_splits=12, test_size=0.1, random_state=42 ).split(X, y):
X_train = X[train_index]
y_train = y[train_index]
X_test = X[val_index]
y_test = y[val_index]
seed = random.randint(0,2<<31)
 np.random.seed(seed)
 logreg = LogisticRegression(dual=False,max_iter=10**5,penalty=penalty, C=C)
 logreg.fit(X_train, y_train)
 score_ += [roc_auc_score(y_train,logreg.predict(X_train)) ]
score = np.mean(score_)
 if score > best_score:
best_std = np.std(score_)
 print(best_score,best_std)
 best_score = score
best_parameters = {'C': C, 'penalty': penalty}
s = seed<train_model>
country=new_data['production_countries'].str.findall(r'[A-Z]{2,5}')
reg_list = []
for train_index, val_index in StratifiedShuffleSplit(n_splits=30, test_size=0.1, random_state=42 ).split(X, y):
X_train = X[train_index]
y_train = y[train_index]
X_test = X[val_index]
y_test = y[val_index]
seed = random.randint(0,2<<31)
 np.random.seed(seed)
 logreg = LogisticRegression(dual=False,max_iter=10**5,**best_parameters)
 logreg.fit(X_train, y_train)
 test_score = logreg.score(X_test, y_test)
 reg_list.append(logreg)
 def auc_reg() :
return sum(map(lambda i:i.score(X_test, y_test), reg_list)) /len(reg_list)
 def score_reg() :
return sum(map(lambda i:roc_auc_score(y_test,i.predict(X_test)) , reg_list)) /len(reg_list)
 def sum_reg(x):
return sum(map(lambda i:i.predict_proba(x), reg_list)) /len(reg_list)
 test_score = score_reg()<prepare_x_and_y>
new_data['production_countries']=country
X_prob = prob_df.drop("target",axis=1)
 Y_prob = prob_df["target"]<compute_test_metric>
new_data['original_and_new']=new_data['original_title']==new_data['title']
sum(map(lambda i:roc_auc_score(Y_prob,i.predict(X_prob)) , reg_list)) /len(reg_list )<compute_train_metric>
new_data=new_data.drop('original_title',axis=1)
 new_data.head()
x = np.average([i.coef_ for i in reg_list],axis = 0)
 print(x,np.std(x),np.median(np.std([i.coef_ for i in reg_list],axis = 0)))
 logreg.coef_ = x
logreg.intercept_ = np.average([i.intercept_ for i in reg_list],axis = 0)
 print(logreg.score(X_test,y_test))
print(roc_auc_score(y_test,logreg.predict(X_test)))
 print(roc_auc_score(y_train,logreg.predict(X_train)) )<compute_test_metric>
new_data=pd.concat([new_data,page],axis=1 )
print(roc_auc_score(Y_prob,logreg.predict(X_prob)) )<compute_test_metric>
new_data['production_companies']=new_data['production_companies'].str.findall(r''name'?:\s?'([A-Za-z]+)')
 new_data.fillna('Unknow')
 print('接下来就是地图可视化了' )
print("Best score: {:.3f} {:.3f}".format(best_score,best_std))
print("Best parameters: {}".format(best_parameters))
print("Best score on test data: {:.3f}".format(test_score))<save_to_csv>
new_data=new_data.drop('imdb_id',axis=1 )
sub = pd.read_csv('.. /input/dont-overfit-ii/sample_submission.csv')
 sub['target'] = sum_reg(robust.transform(test_df)) [:,1]
sub.to_csv('submission.csv', index=False )<load_from_csv>
new_data['spoken_languages']=new_data['spoken_languages'].str.findall(r''([a-z]{2})'' )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 sample_submission = pd.read_csv('.. /input/sample_submission.csv' )<prepare_x_and_y>
new_data['production_companies']=new_data['production_companies'].fillna('unknow' )
target = train['target']<drop_column>
new_data['spoken_languages']=new_data['spoken_languages'].fillna('unknow' )
train = train.drop(['id','target'], axis=1)
 test = test.drop('id', axis=1 )<normalization>
new_data['Keywords']=new_data['Keywords'].str.findall(r''?:\s?'([a-z]+\s?[a-z]+)'' ).fillna('unkonw' )
sc = StandardScaler()<normalization>
d = Counter([i for j in list_of_keywords for i in j] ).most_common(30)
 d
X_train = sc.fit_transform(train)
 X_test = sc.fit_transform(test )<init_hyperparams>
for i in d:
m=i[0]
new_data['Keywords_'+m]=new_data['Keywords'].apply(lambda x:1 if m in x else 0 )
params = {'num_leaves': 9,
'min_data_in_leaf': 42,
'objective': 'binary',
'max_depth': 16,
'learning_rate': 0.0123,
'boosting': 'gbdt',
'bagging_freq': 5,
'bagging_fraction': 0.8,
'feature_fraction': 0.8201,
'bagging_seed': 11,
'reg_alpha': 1.728910519108444,
'reg_lambda': 4.9847051755586085,
'random_state': 42,
'metric': 'auc',
'verbosity': -1,
'subsample': 0.81,
'min_gain_to_split': 0.01077313523861969,
'min_child_weight': 19.428902804238373,
'num_threads': 4}<train_model>
new_data=new_data.drop('Keywords',axis=1 )
train_data = lgb.Dataset(X_train, label=target)
 model = lgb.train(params,
train_data,
num_boost_round=2000,
valid_sets = [train_data],
verbose_eval=500,
early_stopping_rounds = 200)
 y_pred_lgb = model.predict(X_test, num_iteration=model.best_iteration )<train_model>
new_data['cast']=new_data['cast'].fillna('unknow' )
clf = LogisticRegression(class_weight='balanced', solver='liblinear', penalty ='l1', C= 0.1, max_iter=10000)
 clf.fit(X_train, target)
 y_pred_logreg = clf.predict_proba(X_test )<save_to_csv>
for i in d:
m=i[0][0]
new_data['cast_name_'+m]=new_data['cast'].apply(lambda x:1 if m in x else 0 )
sample_submission['target'] = y_pred_lgb
sample_submission.to_csv('submission_lgb.csv', index=False)
 sample_submission['target'] = y_pred_logreg[:,1]
sample_submission.to_csv('submission_logreg.csv', index=False)
<set_options>
list_cast_gender=list(new_data['cast'].str.findall(r''gender'\s?:\s?(\d+)\s?'))
Counter([i for j in list_cast_gender for i in j] ).most_common()
%matplotlib inline
sns.set()
print(os.listdir(".. /input"))
warnings.filterwarnings("ignore", category=DeprecationWarning)
 warnings.filterwarnings("ignore", category=UserWarning)
 warnings.filterwarnings("ignore", category=FutureWarning)
 warnings.filterwarnings("ignore", category=RuntimeWarning )<load_from_csv>
new_data['cast_gender_0']=new_data['cast'].str.findall(r''gender'\s?:\s?(\d+)\s?' ).apply(lambda x: x.count('0'))
new_data['cast_gender_1']=new_data['cast'].str.findall(r''gender'\s?:\s?(\d+)\s?' ).apply(lambda x: x.count('1'))
new_data['cast_gender_2']=new_data['cast'].str.findall(r''gender'\s?:\s?(\d+)\s?' ).apply(lambda x: x.count('2'))
new_data['budget_gender_count']=new_data['budget']/(new_data['cast_gender_0']+new_data['cast_gender_1']+new_data['cast_gender_2']+1 )
train = pd.read_csv(".. /input/train.csv", index_col=0)
 train.shape<load_from_csv>
for g in top_cast_char:
m=g[0]
new_data['cast_char_'+m]=new_data['cast'].apply(lambda x:1 if m in x else 0 )
test = pd.read_csv(".. /input/test.csv", index_col=0)
 test.shape<load_from_csv>
new_data=new_data.drop('cast',axis=1 )
submission = pd.read_csv(".. /input/sample_submission.csv")
 submission.head()<feature_engineering>
new_data['status']=new_data['status'].fillna('Unknow' )
train[train.target==0].shape[0] / train.shape[0]<prepare_x_and_y>
new_data['overview']=new_data['overview'].fillna('' )
X = train.drop("target", axis=1 ).values
y = train.target.values<init_hyperparams>
from sklearn.linear_model import LinearRegression
import eli5
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
n_splits = 25
solver = "liblinear"
penalty = "l1"
C=0.1
class_weight="balanced"<train_on_grid>
log_re=np.log1p(train['revenue'])
 v1=TfidfVectorizer(sublinear_tf=True,analyzer='word',
token_pattern=r'\w{1,}',
ngram_range=(1, 2),
min_df=5)
 overview_text = v1.fit_transform(train['overview'].fillna(''))
linre=LinearRegression()
linre.fit(overview_text,log_re)
skf= StratifiedKFold(n_splits=n_splits, random_state=0)
 lr = LogisticRegression(solver=solver, penalty=penalty, C=C, class_weight=class_weight)
 scores = []
for train_idx, test_idx in skf.split(X, y):
x_train, x_test = X[train_idx], X[test_idx]
y_train, y_test = y[train_idx], y[test_idx]
lr.fit(x_train, y_train)
 proba = lr.predict_proba(x_test)
 scores.append(roc_auc_score(y_test, proba[:,1]))<train_model>
eli5.show_weights(linre, vec=v1, top=20, feature_filter=lambda x: x !='<BIAS>' )
to_try = train.drop("target", axis=1)
 mean_drop_scores = []
std_drop_scores = []
for feat in importance[0:-1]:
to_try = to_try.drop(str(feat), axis=1)
 X = to_try.values
lr = LogisticRegression(solver=solver, C=C, penalty=penalty, class_weight=class_weight)
 stk = StratifiedKFold(n_splits=n_splits, random_state=0)
 scores = []
for train_idx, test_idx in stk.split(X, y):
x_train, x_test = X[train_idx], X[test_idx]
y_train, y_test = y[train_idx], y[test_idx]
lr.fit(x_train, y_train)
 proba = lr.predict_proba(x_test)
 scores.append(roc_auc_score(y_test, proba[:,1]))
mean_score = np.mean(scores)
 std_score = np.std(scores)
 mean_drop_scores.append(mean_score)
 std_drop_scores.append(std_score )<data_type_conversions>
new_data['crew_0']=new_data['crew'].fillna('' ).str.replace(',','' ).str.replace('}','' ).str.findall(''gender\S?'\s?:\s?\S?(\d+)\s?\S?' ).apply(lambda x:x.count('0'))
new_data['crew_1']=new_data['crew'].fillna('' ).str.replace(',','' ).str.replace('}','' ).str.findall(''gender\S?'\s?:\s?\S?(\d+)\s?\S?' ).apply(lambda x:x.count('1'))
new_data['crew_2']=new_data['crew'].fillna('' ).str.replace(',','' ).str.replace('}','' ).str.findall(''gender\S?'\s?:\s?\S?(\d+)\s?\S?' ).apply(lambda x:x.count('2'))
important_features = importance[250::]
important_features = important_features.astype(np.str)
 important_features<define_variables>
for i in d:
new_data['crew_depart_is_'+i[0]]=new_data['crew'].fillna('' ).apply(lambda x:1 if i[0] in x else 0)
 for i in dd:
new_data['crew_job_is_'+i[0]]=new_data['crew'].fillna('' ).apply(lambda x:1 if i[0] in x else 0 )
chris_important_features = ["239", "217", "209", "199", "189",
"164", "117", "108", "91", "73",
"65", "63", "45", "33", "16"]<categorify>
new_data['crew']=new_data['crew'].str.replace(',','' ).str.replace('}','' ).str.replace(r''gender'\s?\S?:\s?\S?\s+'id'\s?:\s?\d+','' ).str.findall(r''department'\s?\S?:\s+\S?(\S+)\S?\s?\S?\s+'job\S?'\s?:\s?'(\D+)\S\s?\S?'name\S?'\s?:\s?'(\D+)'\s?\S?'profile_path\S?':\s+'?(\S+)\S?\s?\s?\S?' )
class MySemiGMM:
def __init__(self, mu, sigma):
self.sigma = sigma
self.mu = mu
self.old_mus = []
self.old_sigmas = []
self.old_pis = []
self.train_roc_convergence = []
self.pi = [0.36, 0.64]
def set_data(self, x_test, x_train, y_train):
self.x_test = x_test
self.y_train = y_train
self.x_train = x_train
self.hot_data = x_train[np.where(y_train==1)]
self.cold_data = x_train[np.where(y_train==0)]
self.n_hot = len(self.hot_data)
 self.n_cold = len(self.cold_data)
 def update_Meff(self):
self.Meff = np.sum(self.gamma, axis=0)
 def update_pars(self):
denominator_cold = self.n_cold + self.Meff[0]
counter_cold = np.sum(self.cold_data, axis=0)
 counter_cold += np.einsum('id,i -> d', self.x_test, self.gamma[:,0])
 mu_cold = counter_cold/denominator_cold
denominator_hot = self.n_hot + self.Meff[1]
counter_hot = np.sum(self.hot_data, axis=0)
 counter_hot += np.einsum('id,i -> d', self.x_test, self.gamma[:,1])
 mu_hot = counter_hot/denominator_hot
self.mu[0] = mu_cold
self.mu[1] = mu_hot
cold_var = np.sum(np.power(self.x_train - self.mu[0], 2), axis=0)
 cold_var += np.einsum('id,i -> d', np.power(self.x_test-self.mu[0], 2), self.gamma[:,0])
 sigma_cold = np.sqrt(1/(self.n_cold + self.Meff[0])* cold_var)
 hot_var = np.sum(np.power(self.x_train - self.mu[1], 2), axis=0)
 hot_var += np.einsum('id,i -> d', np.power(self.x_test-self.mu[1], 2), self.gamma[:,1])
 sigma_hot = np.sqrt(1/(self.n_hot + self.Meff[1])* hot_var)
 self.sigma[0] = sigma_cold
self.sigma[1] = sigma_hot
self.pi[0] =(self.n_cold + self.Meff[0])/(self.x_train.shape[0] + self.x_test.shape[0])
 self.pi[1] =(self.n_hot + self.Meff[1])/(self.x_train.shape[0] + self.x_test.shape[0])
 def learn(self, n_iter=10):
self.old_mus.append(self.mu.copy())
 self.old_sigmas.append(self.sigma.copy())
 self.old_pis.append(self.pi.copy())
 for n in range(n_iter):
self.gamma = self.get_gamma(self.x_test)
 self.update_Meff()
self.update_pars()
self.old_mus.append(self.mu.copy())
 self.old_sigmas.append(self.sigma.copy())
 self.old_pis.append(self.pi.copy())
 score = roc_auc_score(self.y_train, self.predict(self.x_train)[:,1])
 if n % 1 == 0:
print(n, score)
 self.train_roc_convergence.append(score)
 return self.mu
def get_single_normal_proba(self, x, my_mu, my_sigma, my_pi):
sigma2 = np.power(my_sigma,2)
 return my_pi * np.prod(1/np.sqrt(2*np.pi*sigma2)* np.exp(-0.5 *(1/sigma2)* np.power(( x-my_mu),2)))
 def get_gamma(self, X):
probabilities = np.zeros(shape=(X.shape[0], 2))
for n in range(X.shape[0]):
norm_value = self.get_single_normal_proba(X[n,:], self.mu[0], self.sigma[0], self.pi[0])
 norm_value += self.get_single_normal_proba(X[n,:], self.mu[1], self.sigma[1], self.pi[1])
 for m in range(len(self.mu)) :
probabilities[n,m] = self.get_single_normal_proba(X[n,:], self.mu[m], self.sigma[m], self.pi[m])
 probabilities[n,m] /= norm_value
return probabilities
def predict(self, X):
return self.get_gamma(X )<drop_column>
new_data['len_crew']=new_data['crew'].fillna('1' ).apply(lambda x:len(x))
important_features = train.drop("target", axis=1 ).columns.values<find_best_model_class>
new_train=new_data.loc[np.array(train.index)]
m1 = train.loc[train.target==0, important_features].mean().values
m2 = train.loc[train.target==1, important_features].mean().values
mu = np.array([m1, m2])
 old_mu = mu.copy()
s1 = train.loc[train.target==0, important_features].std().values
s2 = train.loc[train.target==1, important_features].std().values
sigma = np.array([s1,s2])
 old_sigma = sigma.copy()
x_train = train.loc[:, important_features].values
y_train = train.target.values
x_test = test.loc[:, important_features].values
model = MySemiGMM(mu, sigma)
 model.set_data(x_test, x_train, y_train)
 new_mu = model.learn()
proba_test = model.predict(x_test)
 proba_train = model.predict(x_train);<find_best_model_class>
new_train['production_countries']=new_train['production_countries'].fillna('' )
m1 = train.loc[train.target==0, chris_important_features].mean().values
m2 = train.loc[train.target==1, chris_important_features].mean().values
mu = np.array([m1, m2])
 old_mu = mu.copy()
s1 = train.loc[train.target==0, chris_important_features].std().values
s2 = train.loc[train.target==1, chris_important_features].std().values
sigma = np.array([s1,s2])
 old_sigma = sigma.copy()
x_train = train.loc[:, chris_important_features].values
y_train = train.target.values
x_test = test.loc[:, chris_important_features].values
model = MySemiGMM(mu, sigma)
 model.set_data(x_test, x_train, y_train)
 new_mu = model.learn()
proba_test = model.predict(x_test)
 proba_train = model.predict(x_train);<import_modules>
new_train['production_countries']=new_train['production_countries'].apply(lambda x:'_'.join(x))
import pandas as pd
import numpy as np
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import GridSearchCV, cross_val_score
from sklearn.feature_selection import RFE
import xgboost as xgb
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import roc_auc_score<load_from_csv>
count=new_train['production_countries'].value_counts()
count=count[count>5]
train = pd.read_csv(".. /input/train.csv" ).drop('id', axis=1)
 test = pd.read_csv('.. /input/test.csv' )<prepare_x_and_y>
bud=new_train.groupby('production_countries' ).budget.mean()
y_train = train['target']
X_train = train.drop('target', axis=1)
 X_test = test.drop('id', axis = 1 )<normalization>
bud=bud.sort_values(ascending=False)[:10]
std = StandardScaler()
X_train = std.fit_transform(X_train)
 X_test = std.fit_transform(X_test )<choose_model_class>
rev=new_train.groupby('production_countries' ).revenue.mean()
xgb_model = xgb.XGBClassifier(random_state=42)
 param_grid = {'objective':['binary:logistic'],
'learning_rate': [0.001,0.05,0.1, 10],
'max_depth': [2,3,4,5,6],
'min_child_weight': [11],
'subsample': [0.8],
'colsample_bytree': [0.7],
'n_estimators': [1000]}
grid = GridSearchCV(estimator = xgb_model, cv=5, param_grid = param_grid , scoring = 'roc_auc', verbose = 1, n_jobs = -1, refit=True)
 grid.fit(X_train,y_train)
 print("Best Score:" + str(grid.best_score_))
print("Best Parameters: " + str(grid.best_params_))
best_parameters = grid.best_params_<find_best_model_class>
rev=rev.sort_values(ascending=False)[:10]
xgb_model = xgb.XGBClassifier(**best_parameters)
 xgb_model.fit(X_train,y_train)
 selector = RFE(xgb_model, 200, step=1)
 selector.fit(X_train,y_train)
 xgb_preds = selector.predict_proba(X_test)[:,1]
train_predict = selector.predict(X_train)
 roc_auc_score(y_train, train_predict )<train_on_grid>
new_train['production_countries'].loc[new_train['production_countries']=='ET']='Ethiopia'
along_co=new_train[new_train['production_countries'].apply(lambda x:1 if len(x)==2 else 0)==1]
along_co.head()
lr = LogisticRegression(random_state=42)
 param_grid = {'class_weight' : ['balanced', None],
'penalty' : ['l2','l1'],
'C' : [0.001, 0.01, 0.1, 1, 10, 100],
'solver': ['saga']}
grid = GridSearchCV(estimator = lr, cv=5, param_grid = param_grid , scoring = 'roc_auc', verbose = 1, n_jobs = -1)
 grid.fit(X_train,y_train)
 print("Best Score:" + str(grid.best_score_))
print("Best Parameters: " + str(grid.best_params_))
best_parameters = grid.best_params_<find_best_model_class>
count_rev=along_co[['production_countries','revenue']]
lr = LogisticRegression(**best_parameters)
 lr.fit(X_train,y_train)
 selector = RFE(lr, 200, step=1)
 selector.fit(X_train,y_train)
 lr_preds = lr.predict_proba(X_test)[:,1]
train_predict = lr.predict(X_train)
 roc_auc_score(y_train, train_predict )<save_to_csv>
dd=count_rev['production_countries'].unique()
mn=pd.DataFrame(dd,columns=['address'])
 mn.head()
final_preds =(lr_preds * 0.8 + xgb_preds * 0.2)
 submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['target'] = final_preds
submission.to_csv('submission.csv', index=False)
 submission.head(20 )<train_model>
import geopandas as ge
x_train=train.drop(['target','id'],axis=1)
 y_train=train['target']
scaler=StandardScaler()
x_train=scaler.fit_transform(x_train)
 model=LogisticRegression(class_weight='balanced', solver='liblinear', penalty ='l1', C= 0.1, max_iter=10000)
 model.fit(x_train,y_train)
 print("Score on training data: " + str(model.score(x_train,y_train)*100)+ "%" )<save_to_csv>
world = ge.read_file(ge.datasets.get_path('naturalearth_lowres'))
x_test=test.drop('id',axis=1)
 x_test=scaler.fit_transform(x_test)
 y_test=model.predict_proba(x_test)
 print(y_test)
 ans=pd.DataFrame({'id':test['id'],'target':y_test[:,1]})
 ans.to_csv('output.csv',index=False )<load_from_csv>
from mpl_toolkits.axes_grid1 import make_axes_locatable
print(os.listdir(".. /input"))
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 targets = train['target']
train = train.drop(columns=['id', 'target'])
 ids = test['id']
test = test.drop(columns=['id'] )<prepare_x_and_y>
dd=count_rev['production_countries'].value_counts()
mn['values']=dd.values
mn.head()
transformer = RobustScaler().fit(test.values)
 trans_data = transformer.transform(train.values)
 trans_test = transformer.transform(test.values)
 trans_data_df = pd.DataFrame(trans_data, columns=train.columns)
 trans_test_df = pd.DataFrame(trans_test, columns=test.columns )<train_model>
mn['address'][0]='United States'
mn['address'][2]='Korea'
mn['address'][1]='India'
mn['address'][3]='Serbia'
mn['address'][4]='United Kingdom'
mn['address'][5]='France'
mn['address'][6]='New Zealand'
mn['address'][7]='Italy'
mn['address'][8]='Belgium'
mn['address'][9]='Czech Rep.'
mn['address'][11]='Russia'
mn['address'][12]='Spain'
mn['address'][13]='Turkey'
mn['address'][14]='China'
mn['address'][15]='Canada'
mn['address'][16]='Australia'
mn['address'][17]='Iran'
mn['address'][18]='Japan'
mn['address'][19]='Sweden'
mn['address'][20]='Philippines'
mn['address'][21]='Brazil'
mn['address'][22]='Netherlands'
mn['address'][23]='Ireland'
mn['address'][24]='Mexico'
mn['address'][25]='Germany'
mn['address'][26]='South Africa'
mn['address'][27]='Finland'
mn['address'][28]='Pakistan'
mn['address'][29]='Norway'
mn['address'][30]='Bulgaria'
mn['address'][31]='Ukraine'
mn['address'][32]='Denmark'
mn['address'][33]='Romania'
mn['address'][35]='Hungary'
mn['address'][36]='Chile'
mn['address'][37]='Indonesia'
mn['address'][38]='Poland'
center_point_0 = trans_data_df[targets == 0].mean()
center_point_1 = trans_data_df[targets == 1].mean()
start = np.array([center_point_0, center_point_1])
 kmeans = KMeans(n_clusters=2, init=start ).fit(test.values)
 g_mix = GaussianMixture(n_components=2 ).fit(test.values)
<predict_on_test>
mn=mn.drop(10)
 mn=mn.drop(34 )
predicts = kmeans.predict(train.values)
 g_predicts= g_mix.predict(train.values)
 count= 0
for i in range(len(predicts)) :
if targets[i] == predicts[i]:
count+=1
print("K-Means Cluster Matches Target : ", str(count), " times out of 250")
 data = pd.DataFrame(trans_data.copy() , columns=train.columns)
 data['k_cluster'] = predicts
data['g_mix'] = g_predicts
t_data = pd.DataFrame(trans_test.copy() , columns=test.columns)
 t_data['k_cluster'] = kmeans.predict(test.values)
 t_data['g_mix'] = g_mix.predict(test.values )<filter>
mn['geometry']='unknow'
count=0
for i in range(len(g_predicts)) :
if targets[i] == g_predicts[i]:
count+=1
print("For Gaussian-Mixture : ", count )<save_to_csv>
for i in range(39):
if i==10 or i==34:
continue;
d=world[world['name']==mn['address'][i]]['geometry'].values[0]
mn['geometry'][i]=d
param_grid = {"C": [.01,.1, 1, 10, 100, 1000],
"penalty":('l1','l2')}
top_data = data.drop(columns=['k_cluster', 'g_mix'])
 clf = LogisticRegression(random_state=0, solver='liblinear', class_weight='balanced', max_iter = 1000)
 gs = GridSearchCV(clf, param_grid, cv=5)
 gs.fit(top_data.values, targets)
 print(gs.best_score_)
 print(gs.best_estimator_)
 predictions = gs.predict_proba(t_data.drop(columns=['k_cluster', 'g_mix'] ).values)[:,1]
output = pd.DataFrame({'id' : ids, 'target' : predictions})
 output.to_csv('scale_only_output.csv',index=None )<save_to_csv>
TS = train.loc[:,["original_title","release_date","budget","runtime","revenue"]]
TS.dropna()
TS.release_date = pd.to_datetime(TS.release_date)
 TS.loc[:,"Year"] = TS["release_date"].dt.year
TS.loc[:,"Month"] = TS["release_date"].dt.month
TS = TS[TS.Year<2018]
param_grid = {"C": [.01,.1, 1, 10, 100, 1000],
"penalty":('l1','l2')}
top_data = data
clf = LogisticRegression(random_state=0, solver='liblinear', class_weight='balanced', max_iter = 1000)
 gs = GridSearchCV(clf, param_grid, cv=5)
 gs.fit(top_data.values, targets)
 print(gs.best_score_)
 print(gs.best_estimator_)
 predictions = gs.predict_proba(t_data.values)[:,1]
output = pd.DataFrame({'id' : ids, 'target' : predictions})
 output.to_csv('scale_and_clustering_output.csv',index=None )<save_to_csv>
top3 = train.sort_values(by='popularity',ascending=False)[:10]
id3=top3[['title','poster_path','revenue']]
param_grid = {"C": [.01,.1, 1, 10, 100, 1000],
"penalty":('l1','l2')}
top_data = data[labels]
clf = LogisticRegression(random_state=0, solver='liblinear', class_weight='balanced', max_iter = 1000)
 gs = GridSearchCV(clf, param_grid, cv=5)
 gs.fit(top_data.values, targets)
 print(gs.best_score_)
 print(gs.best_estimator_)
 predictions = gs.predict_proba(t_data[labels].values)[:,1]
output = pd.DataFrame({'id' : ids, 'target' : predictions})
 output.to_csv('reduced_scale_and_clustering_output.csv',index=None )<import_modules>
old_title=new_data['title'].value_counts()
import numpy as np
import pandas as pd
import os<load_from_csv>
a=old_title[old_title.values==3].index
b=old_title[old_title==2].index
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 targets = train['target']
train = train.drop(columns=['id', 'target'])
 ids = test['id']
test = test.drop(columns=['id'])
 transformer = RobustScaler().fit(train.values)
 trans_x = transformer.transform(train.values)
 trans_test = transformer.transform(test.values )<train_on_grid>
new_data['fan_pai_2']=new_data['title'].apply(lambda x:1 if x in a else 0)
 new_data['pan_pai_3']=new_data['title'].apply(lambda x:1 if x in b else 0 )
scores = []
param_grid = {"C": [.01,.1, 1, 10, 100, 1000],
"penalty":('l1','l2')}
clf = LogisticRegression(random_state=0, solver='liblinear', class_weight='balanced', max_iter = 1000)
 gs = GridSearchCV(clf, param_grid, cv=5)
 gs.fit(trans_x, targets)
 best_score = gs.best_score_
best_estimator = gs.best_estimator_
best_pca = 'N/a'
print(best_score)

pca = PCA(n_components=2)
 pca.fit(train.values)
 vals_2d = pca.transform(train.values)
 colors = ['red' if t == 1 else 'blue' for t in targets]
for i in range(len(vals_2d)) :
plt.scatter(x=vals_2d[i,0], y=vals_2d[i,1], color=colors[i])
<categorify>
new_data.loc[new_data['title'].fillna('un' ).str.contains('Planet of the Apes')]
selector = RFECV(best_estimator, step=1, cv=5)
 selector = selector.fit(trans_x, targets)
 new_trans_x = selector.transform(trans_x)
 new_trans_test = selector.transform(trans_test)
<train_on_grid>
top10 = train.sort_values(by='revenue',ascending=False)[:10]
id10=top10[['title','poster_path','revenue']]
param_grid = {"C": [.01,.1],
"penalty":('l1','l2')}
clf = LogisticRegression(random_state=0, solver='liblinear', class_weight='balanced', max_iter = 1000)
 gs = GridSearchCV(clf, param_grid, cv=5)
 gs.fit(new_trans_x, targets)
 print(gs.best_estimator_)
 print(gs.best_score_)
 print(gs.score(new_trans_x, targets))
predictions = gs.predict_proba(new_trans_test)[:,1]<save_to_csv>
title_count=train['title'].value_counts()
len(title_count[title_count.values!=1] )
output = pd.DataFrame({'id' : ids, 'target' : predictions})
 output.to_csv('output.csv',index=None )<import_modules>
train[train['title'].str.contains('Furious')]['title']
import pandas as pd
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt<load_from_csv>
test[test['title'].fillna('Unknow' ).str.contains('Furious')]['title']
train_df = pd.read_csv(".. /input/train.csv" ).drop('id', axis=1)
 train_df.head()<load_from_csv>
d.remove(d[7] )
test_df = pd.read_csv('.. /input/test.csv' ).drop('id', axis = 1)
 test_df.head()<prepare_x_and_y>
for i in d :
new_data['crew_contains_'+i[0][2]]=new_data['crew'].fillna('' ).apply(lambda x:1 if i[0][2] in x else 0)
y = train_df['target']
X = train_df.drop('target', axis=1 )<split>
mon_is=[1,6,7,9,10,12]
for i in mon_is:
new_data['mon_is_'+str(i)]=new_data['release_month'].apply(lambda x:1 if x==i else 0 )
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42 )<train_on_grid>
new_data['spoken_languages_count']=new_data['spoken_languages'].apply(lambda x:len(x))
best_score = 0
for penalty in ['l1', 'l2']:
for C in [0.001, 0.01, 0.1, 1, 10, 100]:
logreg = LogisticRegression(class_weight='balanced', penalty=penalty, C=C, solver='liblinear')
 logreg.fit(X_train, y_train)
 score = logreg.score(X_test, y_test)
 if score > best_score:
best_score = score
best_parameters = {'C': C, 'penalty': penalty}<train_model>
new_data.loc[train.index].groupby('spoken_languages_count' ).revenue.median()
logreg = LogisticRegression(**best_parameters)
 logreg.fit(X_train, y_train)
 test_score = logreg.score(X_test, y_test )<compute_test_metric>
status_du=pd.get_dummies(new_data['status'] )
print("Best score: {:.2f}".format(best_score))
print("Best parameters: {}".format(best_parameters))
print("Best score on test data: {:.2f}".format(test_score))<save_to_csv>
new_data=pd.concat([new_data,status_du],axis=1)
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['target'] = logreg.predict_proba(test_df)[:,1]
sub.to_csv('submission.csv', index=False )<load_from_csv>
new_data=new_data.drop(['status'],axis=1)
 new_data.head()
pdf=pd.read_csv(".. /input/mergeneural512newparamsless/mergeNeuralNTPost99.csv")
 pdf.shape 
<feature_engineering>
new_data=new_data.drop(['original_language'],axis=1 )
def alterProbabilities(odf, sqrt=False):
df=copy.deepcopy(odf)
 for cindex in df.columns:
df['adjSum']=0
if cindex != 'object_id':
df.loc[:,'not_' + str(cindex)]=1.0 - df.loc[:,cindex]
zeroFilter=(df.loc[:,'not_' + str(cindex)]==0)
 df.loc[zeroFilter,'adj_' + str(cindex)]=999999
df.loc[~zeroFilter,'adj_' + str(cindex)]=df.loc[~zeroFilter,cindex] / df.loc[~zeroFilter,'not_' + str(cindex)]
if sqrt:
df['adj_' + str(cindex)]=np.sqrt(df['adj_' + str(cindex)])
 for cindex in odf.columns:
if cindex !='object_id':
df.loc[:,'adjSum']+=df.loc[:,'adj_' + str(cindex)]
for cindex in odf.columns:
if cindex !='object_id':
odf.loc[:,cindex]=df.loc[:,'adj_' + str(cindex)] / df.loc[:,'adjSum']
return odf, df
s, os=alterProbabilities(pdf)
 s.head() 
<prepare_x_and_y>
sum(new_data['geres_name'].isna() )
def GenUnknown(data):
return(((((( data["mymedian"])+(((data["mymean"])/ 2.0)))/2.0)) +(((((1.0)-(((data["mymax"])*(((data["mymax"])*(data["mymax"])))))))/ 2.0)))/2.0)
 feats = ['class_6', 'class_15', 'class_16', 'class_42', 'class_52', 'class_53',
'class_62', 'class_64', 'class_65', 'class_67', 'class_88', 'class_90',
'class_92', 'class_95']
y = pd.DataFrame()
y['mymean'] = s[feats].mean(axis=1)
 y['mymedian'] = s[feats].median(axis=1)
 y['mymax'] = s[feats].max(axis=1)
 s['class_99'] = GenUnknown(y )<save_to_csv>
new_data['geres_name']=new_data['geres_name'].fillna('Unknow' )
s.to_csv('moreDecisiveNeural.csv', index=False )<set_options>
new_data['geres_name_count']=new_data['geres_name'].apply(lambda x:len(x))
pd.options.display.precision = 15
get_ipython().run_line_magic('matplotlib', 'inline')
<load_from_csv>
ge_name_count_rev=new_data.loc[train.index].groupby(['geres_name_count'] ).revenue.median().sort_values()
train_dataset = pd.read_csv('.. /input/train.csv', dtype={'acoustic_data': np.int16, 'time_to_failure': np.float32} )<prepare_x_and_y>
new_data=new_data.drop('homepage_end',axis=1 )
instances = 150000
segments = int(np.floor(train_dataset.shape[0] / instances))
X_trainset = pd.DataFrame(index=range(segments), dtype=np.float64)
 y_trainset = pd.DataFrame(index=range(segments), dtype=np.float64,
columns=['time_to_failure'])
 for segment in tqdm_notebook(range(segments)) :
each_seg = train_dataset.iloc[segment * instances : segment * instances + instances]
x_rawdata = each_seg['acoustic_data']
x = x_rawdata.values
y = each_seg['time_to_failure'].values[-1]
y_trainset.loc[segment, 'time_to_failure'] = y
X_trainset.loc[segment, 'average'] = x.mean()
X_trainset.loc[segment, 'standard_deviation'] = x.std()
X_trainset.loc[segment, 'maximum'] = x.max()
X_trainset.loc[segment, 'minimum'] = x.min()
X_trainset.loc[segment, 'quantile_1_percentile'] = np.quantile(x,0.01)
 X_trainset.loc[segment, 'quantile_5_percentile'] = np.quantile(x,0.05)
 X_trainset.loc[segment, 'quantile_95_percentile'] = np.quantile(x,0.95)
 X_trainset.loc[segment, 'quantile_99_percentile'] = np.quantile(x,0.99)
 X_trainset.loc[segment, 'median_absolute'] = np.median(np.abs(x))
X_trainset.loc[segment, 'quantile_95_percentile_absolute'] = np.quantile(np.abs(x),0.95)
 X_trainset.loc[segment, 'quantile_99_percentile_absolute'] = np.quantile(np.abs(x),0.99)
 X_trainset.loc[segment, 'F_test_measure'], X_trainset.loc[segment, 'p_test_measure'] = stats.f_oneway(x[:30000],x[30000:60000],x[60000:90000],x[90000:120000],x[120000:])
 X_trainset.loc[segment, 'average_change_absolute'] = np.mean(np.diff(x))
X_trainset.loc[segment, 'average_change_rate'] = np.mean(np.nonzero(( np.diff(x)/ x[:-1])) [0])
 X_trainset.loc[segment, 'maximum_absolute'] = np.abs(x ).max()
for windows in [10,100]:
x_roll_std = x_rawdata.rolling(windows ).std().dropna().values
x_roll_mean = x_rawdata.rolling(windows ).mean().dropna().values
X_trainset.loc[segment, 'average_rolling_standard_deviation' + str(windows)] = x_roll_std.mean()
X_trainset.loc[segment, 'standard_deviation_rolling_standard_deviation' + str(windows)] = x_roll_std.std()
X_trainset.loc[segment, 'maximum_rolling_standard_deviation' + str(windows)] = x_roll_std.max()
X_trainset.loc[segment, 'minimum_rolling_standard_deviation' + str(windows)] = x_roll_std.min()
X_trainset.loc[segment, 'quantile_1_percentile_rolling_standard_deviation' + str(windows)] = np.quantile(x_roll_std,0.01)
 X_trainset.loc[segment, 'quantile_5_percentile_rolling_standard_deviation' + str(windows)] = np.quantile(x_roll_std,0.05)
 X_trainset.loc[segment, 'quantile_95_percentile_rolling_standard_deviation' + str(windows)] = np.quantile(x_roll_std,0.95)
 X_trainset.loc[segment, 'quantile_99_percentile_rolling_standard_deviation' + str(windows)] = np.quantile(x_roll_std,0.99)
 X_trainset.loc[segment, 'average_change_absolute_rolling_standard_deviation' + str(windows)] = np.mean(np.diff(x_roll_std))
X_trainset.loc[segment, 'average_change_rate_rolling_standard_deviation' + str(windows)] = np.mean(np.nonzero(( np.diff(x_roll_std)/ x_roll_std[:-1])) [0])
 X_trainset.loc[segment, 'maximum_absolute_rolling_standard_deviation' + str(windows)] = np.abs(x_roll_std ).max()
X_trainset.loc[segment, 'average_absolute_rolling_mean' + str(windows)] = x_roll_mean.mean()
X_trainset.loc[segment, 'standard_deviation_rolling_mean' + str(windows)] = x_roll_mean.std()
X_trainset.loc[segment, 'maximum_rolling_mean' + str(windows)] = x_roll_mean.max()
X_trainset.loc[segment, 'minimum_rolling_mean' + str(windows)] = x_roll_mean.min()
X_trainset.loc[segment, 'quantile_1_percentile_rolling_mean' + str(windows)] = np.quantile(x_roll_mean,0.01)
 X_trainset.loc[segment, 'quantile_5_percentile_rolling_mean' + str(windows)] = np.quantile(x_roll_mean,0.05)
 X_trainset.loc[segment, 'quantile_95_percentile_rolling_mean' + str(windows)] = np.quantile(x_roll_mean,0.95)
 X_trainset.loc[segment, 'quantile_99_percentile_rolling_mean' + str(windows)] = np.quantile(x_roll_mean,0.99)
 X_trainset.loc[segment, 'average_change_absolute_rolling_mean' + str(windows)] = np.mean(np.diff(x_roll_mean))
X_trainset.loc[segment, 'average_change_rate_rolling_mean' + str(windows)] = np.mean(np.nonzero(( np.diff(x_roll_mean)/ x_roll_mean[:-1])) [0])
 X_trainset.loc[segment, 'maximum_absolute_rolling_mean' + str(windows)] = np.abs(x_roll_mean ).max() 
<load_from_csv>
new_data['original_and_new']=new_data['original_and_new'].apply(lambda x: 1 if x else 0 )
submission = pd.read_csv('.. /input/sample_submission.csv', index_col='seg_id')
 X_testset = pd.DataFrame(columns=X_trainset.columns, dtype=np.float64, index=submission.index)
 for i, seg_id in enumerate(tqdm_notebook(X_testset.index)) :
each_seg = pd.read_csv('.. /input/test/' + seg_id + '.csv')
 x_rawdata = each_seg['acoustic_data']
x_roll = x_rawdata.rolling(windows ).std().dropna().values
x = x_rawdata.values
X_testset.loc[seg_id, 'average'] = x.mean()
X_testset.loc[seg_id, 'standard_deviation'] = x.std()
X_testset.loc[seg_id, 'maximum'] = x.max()
X_testset.loc[seg_id, 'minimum'] = x.min()
X_testset.loc[seg_id, 'quantile_1_percentile'] = np.quantile(x,0.01)
 X_testset.loc[seg_id, 'quantile_5_percentile'] = np.quantile(x,0.05)
 X_testset.loc[seg_id, 'quantile_95_percentile'] = np.quantile(x,0.95)
 X_testset.loc[seg_id, 'quantile_99_percentile'] = np.quantile(x,0.99)
 X_testset.loc[seg_id, 'median_absolute'] = np.median(np.abs(x))
X_testset.loc[seg_id, 'quantile_95_percentile_absolute'] = np.quantile(np.abs(x),0.95)
 X_testset.loc[seg_id, 'quantile_99_percentile_absolute'] = np.quantile(np.abs(x),0.99)
 X_testset.loc[seg_id, 'F_test_measure'], X_trainset.loc[segment, 'p_test_measure'] = stats.f_oneway(x[:30000],x[30000:60000],x[60000:90000],x[90000:120000],x[120000:])
 X_testset.loc[seg_id, 'average_change_absolute'] = np.mean(np.diff(x))
X_testset.loc[seg_id, 'average_change_rate'] = np.mean(np.nonzero(( np.diff(x)/ x[:-1])) [0])
 X_testset.loc[seg_id, 'maximum_absolute'] = np.abs(x ).max()
for windows in [10,100]:
x_roll_std = x_rawdata.rolling(windows ).std().dropna().values
x_roll_mean = x_rawdata.rolling(windows ).mean().dropna().values
X_testset.loc[seg_id, 'average_rolling_standard_deviation' + str(windows)] = x_roll_std.mean()
X_testset.loc[seg_id, 'standard_deviation_rolling_standard_deviation' + str(windows)] = x_roll_std.std()
X_testset.loc[seg_id, 'maximum_rolling_standard_deviation' + str(windows)] = x_roll_std.max()
X_testset.loc[seg_id, 'minimum_rolling_standard_deviation' + str(windows)] = x_roll_std.min()
X_testset.loc[seg_id, 'quantile_1_percentile_rolling_standard_deviation' + str(windows)] = np.quantile(x_roll_std,0.01)
 X_testset.loc[seg_id, 'quantile_5_percentile_rolling_standard_deviation' + str(windows)] = np.quantile(x_roll_std,0.05)
 X_testset.loc[seg_id, 'quantile_95_percentile_rolling_standard_deviation' + str(windows)] = np.quantile(x_roll_std,0.95)
 X_testset.loc[seg_id, 'quantile_99_percentile_rolling_standard_deviation' + str(windows)] = np.quantile(x_roll_std,0.99)
 X_testset.loc[seg_id, 'average_change_absolute_rolling_standard_deviation' + str(windows)] = np.mean(np.diff(x_roll_std))
X_testset.loc[seg_id, 'average_change_rate_rolling_standard_deviation' + str(windows)] = np.mean(np.nonzero(( np.diff(x_roll_std)/ x_roll_std[:-1])) [0])
 X_testset.loc[seg_id, 'maximum_absolute_rolling_standard_deviation' + str(windows)] = np.abs(x_roll_std ).max()
X_testset.loc[seg_id, 'average_absolute_rolling_mean' + str(windows)] = x_roll_mean.mean()
X_testset.loc[seg_id, 'standard_deviation_rolling_mean' + str(windows)] = x_roll_mean.std()
X_testset.loc[seg_id, 'maximum_rolling_mean' + str(windows)] = x_roll_mean.max()
X_testset.loc[seg_id, 'minimum_rolling_mean' + str(windows)] = x_roll_mean.min()
X_testset.loc[seg_id, 'quantile_1_percentile_rolling_mean' + str(windows)] = np.quantile(x_roll_mean,0.01)
 X_testset.loc[seg_id, 'quantile_5_percentile_rolling_mean' + str(windows)] = np.quantile(x_roll_mean,0.05)
 X_testset.loc[seg_id, 'quantile_95_percentile_rolling_mean' + str(windows)] = np.quantile(x_roll_mean,0.95)
 X_testset.loc[seg_id, 'quantile_99_percentile_rolling_mean' + str(windows)] = np.quantile(x_roll_mean,0.99)
 X_testset.loc[seg_id, 'average_change_absolute_rolling_mean' + str(windows)] = np.mean(np.diff(x_roll_mean))
X_testset.loc[seg_id, 'average_change_rate_rolling_mean' + str(windows)] = np.mean(np.nonzero(( np.diff(x_roll_mean)/ x_roll_mean[:-1])) [0])
 X_testset.loc[seg_id, 'maximum_absolute_rolling_mean' + str(windows)] = np.abs(x_roll_mean ).max()<choose_model_class>
list_geres_name=['Comedy','Thriller','Action','Drama','Romance']
num_folds = 5
k_folds = KFold(n_splits=num_folds, shuffle=True, random_state=11 )<split>
for i in list_geres_name:
new_data['geres_name'+'_'+i]=new_data['geres_name'].apply(lambda x: 1 if i in x else 0 )
def train_model_lgb(X=X_trainset, X_testset=X_testset, y=y_trainset, params=None, k_folds=k_folds, model=None):
x_values = np.zeros(len(X))
prediction = np.zeros(len(X_testset))
scores = []
feature_importance = pd.DataFrame()
for fold_n,(trainset_index, valid_set_index)in enumerate(k_folds.split(X)) :
print('Fold', fold_n, 'started at', time.ctime())
 X_train_per_fold, X_valid_per_fold = X.iloc[trainset_index], X.iloc[valid_set_index]
y_train_per_fold, y_valid_per_fold = y.iloc[trainset_index], y.iloc[valid_set_index]
model = lgb.LGBMRegressor(**params, n_estimators = 20000, nthread = 4, n_jobs = -1)
 model.fit(X_train_per_fold, y_train_per_fold,
eval_set=[(X_train_per_fold, y_train_per_fold),(X_valid_per_fold, y_valid_per_fold)], eval_metric='mae',
verbose=1000, early_stopping_rounds=200)
 y_pred_valid = model.predict(X_valid_per_fold)
 y_pred = model.predict(X_testset, num_iteration=model.best_iteration_)
 x_values[valid_set_index] = y_pred_valid.reshape(-1,)
 scores.append(mean_absolute_error(y_valid_per_fold, y_pred_valid))
prediction += y_pred
prediction /= num_folds
print('CV mean score: {0:.4f}.'.format(mean_absolute_error(y, x_values)))
 return x_values, prediction
def train_model_xgb(X=X_trainset, X_testset=X_testset, y=y_trainset, params=None, k_folds=k_folds, model=None):
x_value = np.zeros(len(X))
prediction = np.zeros(len(X_testset))
scores = []
feature_importance = pd.DataFrame()
for fold_n,(trainset_index, valid_set_index)in enumerate(k_folds.split(X)) :
print('Fold', fold_n, 'started at', time.ctime())
 X_train_per_fold, X_valid_per_fold = X.iloc[trainset_index], X.iloc[valid_set_index]
y_train_per_fold, y_valid_per_fold = y.iloc[trainset_index], y.iloc[valid_set_index]
train_data = xgb.DMatrix(data=X_train_per_fold, label=y_train_per_fold, feature_names=X_trainset.columns)
 valid_data = xgb.DMatrix(data=X_valid_per_fold, label=y_valid_per_fold, feature_names=X_trainset.columns)
 watchlist = [(train_data, 'train'),(valid_data, 'valid_data')]
model = xgb.train(dtrain=train_data, num_boost_round=20000, evals=watchlist, early_stopping_rounds=200, verbose_eval=500, params=params)
 y_pred_valid = model.predict(xgb.DMatrix(X_valid_per_fold, feature_names=X_trainset.columns), ntree_limit=model.best_ntree_limit)
 y_pred = model.predict(xgb.DMatrix(X_testset, feature_names=X_trainset.columns), ntree_limit=model.best_ntree_limit)
 x_value[valid_set_index] = y_pred_valid.reshape(-1,)
 scores.append(mean_absolute_error(y_valid_per_fold, y_pred_valid))
prediction += y_pred
prediction /= num_folds
print('CV mean score: {0:.4f}.'.format(mean_absolute_error(y, x_value)))
 return x_value, prediction<init_hyperparams>
new_data=new_data.drop('geres_name',axis=1 )
lgb_params = {'num_leaves': 64,
'min_data_in_leaf': 50,
'objective': 'mae',
'max_depth': -1,
'learning_rate': 0.001,
"boosting": "gbdt",
"feature_fraction": 0.5,
"bagging_freq": 2,
"bagging_fraction": 0.5,
"bagging_seed": 0,
"metric": 'mae',
"verbosity": -1,
'reg_alpha': 1.0,
'reg_lambda': 1.0,
}
x_value_lgb, prediction_lgb = train_model_lgb(params = lgb_params )<init_hyperparams>
new_data['production_countries']=new_data['production_countries'].fillna('QQ' )
xgb_params = {'eta': 0.01,
'max_depth': 6,
'subsample': 0.8,
'colsample_bytree': 0.8,
'colsample_bylevel': 0.8,
'colsample_bynode': 0.8,
'lambda': 0.1,
'alpha' : 0.1,
'objective': 'reg:linear',
'eval_metric': 'mae',
'silent': True,
'nthread': 4}
x_value_xgb, prediction_xgb = train_model_xgb(params=xgb_params )<compute_test_metric>
new_data['production_countries'].loc[new_data['production_countries']=='Unknow']['production_countries']='QQ'
print(mean_absolute_error(y_trainset,(x_value_lgb)) )<save_to_csv>
list_pro_coun=list(new_data['production_countries'])
 d = Counter([j for i in list_pro_coun for j in i] ).most_common(5)
 d
submission['time_to_failure'] =(prediction_lgb)
 print(submission.head())
 submission.to_csv('submission_all.csv' )<import_modules>
for i in d:
new_data['production_cou_name'+'_'+i[0]]=new_data['production_countries'].apply(lambda x: 1 if i[0] in x else 0 )
import matplotlib.pyplot as plt
from tqdm import tqdm
from sklearn.preprocessing import StandardScaler
from sklearn.svm import NuSVR
from sklearn.kernel_ridge import KernelRidge
from sklearn.metrics import mean_absolute_error, make_scorer
from sklearn.model_selection import GridSearchCV<load_from_csv>
new_data['pro_country_count']=new_data['production_companies'].apply(lambda x:len(x))
train = pd.read_csv('.. /input/train.csv', dtype={'acoustic_data': np.int16, 'time_to_failure': np.float64} )<set_options>
pro_count_rev=new_data.loc[train.index].groupby('pro_country_count' ).revenue.median()
pd.options.display.precision = 15
train.head()<train_model>
new_data=new_data.drop('production_countries',axis=1 )
def add_trend_feature(arr, abs_values=False):
idx = np.array(range(len(arr)))
 if abs_values:
arr = np.abs(arr)
 lr = LinearRegression()
lr.fit(idx.reshape(-1, 1), arr)
 return lr.coef_[0] 
<prepare_x_and_y>
new_data['production_companies'].loc[new_data['production_companies']=='unknow']='u'
rows = 150_000
segments = int(np.floor(train.shape[0] / rows))
X_train = pd.DataFrame(index=range(segments), dtype=np.float64,
columns=['ave', 'std', 'max', 'min','q95','q99', 'q05','q01',
'abs_max', 'abs_mean', 'abs_std', 'trend', 'abs_trend', 'iqr',
'q999','q001','ave10'])
 y_train = pd.DataFrame(index=range(segments), dtype=np.float64,
columns=['time_to_failure'])
 for segment in tqdm(range(segments)) :
seg = train.iloc[segment*rows:segment*rows+rows]
x = seg['acoustic_data'].values
y = seg['time_to_failure'].values[-1]
y_train.loc[segment, 'time_to_failure'] = y
X_train.loc[segment, 'ave'] = x.mean()
X_train.loc[segment, 'std'] = x.std()
X_train.loc[segment, 'max'] = x.max()
X_train.loc[segment, 'min'] = x.min()
X_train.loc[segment, 'q95'] = np.quantile(x,0.95)
 X_train.loc[segment, 'q99'] = np.quantile(x,0.99)
 X_train.loc[segment, 'q05'] = np.quantile(x,0.05)
 X_train.loc[segment, 'q01'] = np.quantile(x,0.01)
 X_train.loc[segment, 'abs_max'] = np.abs(x ).max()
X_train.loc[segment, 'abs_mean'] = np.abs(x ).mean()
X_train.loc[segment, 'abs_std'] = np.abs(x ).std()
X_train.loc[segment, 'trend'] = add_trend_feature(x)
 X_train.loc[segment, 'abs_trend'] = add_trend_feature(x, abs_values=True)
 X_train.loc[segment, 'iqr'] = np.subtract(*np.percentile(x, [75, 25]))
X_train.loc[segment, 'q999'] = np.quantile(x,0.999)
 X_train.loc[segment, 'q001'] = np.quantile(x,0.001)
 X_train.loc[segment, 'ave10'] = stats.trim_mean(x, 0.1 )<normalization>
d.remove(( 'u',414))
scaler = StandardScaler()
scaler.fit(X_train)
 X_train_scaled = scaler.transform(X_train )<train_on_grid>
dd=[]
for i in d:
dd.append(i[0] )
scorer = make_scorer(mean_absolute_error, greater_is_better=False)
 parameters = [{ 'gamma': [0.6, 0.7, 0.8],
'C': [2.35, 2.4, 2.45, 2.5],
'nu': [0.85, 0.9, 0.95]}]
reg1 = GridSearchCV(NuSVR(kernel='rbf', tol=0.01), parameters, cv = 3, scoring=scorer)
 reg1.fit(X_train_scaled, y_train.values.flatten())
 y_pred1 = reg1.predict(X_train_scaled)
 print(reg1.best_params_)
 print(reg1.best_score_ )<train_on_grid>
dd=[]
for i in d:
dd.append(i[0] )
parameters = [{ 'gamma': [0.06, 0.1, 0.08, 0.09],
'alpha': [0.005, 0.01, 0.05]}]
reg2 = GridSearchCV(KernelRidge(kernel='rbf'), parameters, cv = 3, scoring=scorer)
 reg2.fit(X_train_scaled, y_train.values.flatten())
 y_pred2 = reg2.predict(X_train_scaled)
 print(reg2.best_params_)
 print(reg2.best_score_ )<compute_test_metric>
new_data=new_data.drop('spoken_languages',axis=1 )
score1 = mean_absolute_error(y_train.values.flatten() , y_pred1)
 print(f'Score1: {score1:0.3f}')
 score2 = mean_absolute_error(y_train.values.flatten() , y_pred2)
 print(f'Score2: {score2:0.3f}')
 score3 = mean_absolute_error(y_train.values.flatten() , y_pred1*0.5+y_pred2*0.5)
 print(f'Score3: {score3:0.3f}' )<load_from_csv>
new_data['runtime'].loc[new_data['runtime']==0]=1
new_data['time_budget']=new_data['budget'].fillna(0)/new_data['runtime'].fillna(1 )
submission = pd.read_csv('.. /input/sample_submission.csv', index_col='seg_id')
 X_test = pd.DataFrame(columns=X_train.columns, dtype=np.float64, index=submission.index)
 for seg_id in X_test.index:
seg = pd.read_csv('.. /input/test/' + seg_id + '.csv')
 x = seg['acoustic_data'].values
X_test.loc[seg_id, 'ave'] = x.mean()
X_test.loc[seg_id, 'std'] = x.std()
X_test.loc[seg_id, 'max'] = x.max()
X_test.loc[seg_id, 'min'] = x.min()
X_test.loc[seg_id, 'q95'] = np.quantile(x,0.95)
 X_test.loc[seg_id, 'q99'] = np.quantile(x,0.99)
 X_test.loc[seg_id, 'q05'] = np.quantile(x,0.05)
 X_test.loc[seg_id, 'q01'] = np.quantile(x,0.01)
 X_test.loc[seg_id, 'abs_max'] = np.abs(x ).max()
X_test.loc[seg_id, 'abs_mean'] = np.abs(x ).mean()
X_test.loc[seg_id, 'abs_std'] = np.abs(x ).std()
X_test.loc[seg_id, 'trend'] = add_trend_feature(x)
 X_test.loc[seg_id, 'abs_trend'] = add_trend_feature(x, abs_values=True)
 X_test.loc[seg_id, 'iqr'] = np.subtract(*np.percentile(x, [75, 25]))
X_test.loc[seg_id, 'q999'] = np.quantile(x,0.999)
 X_test.loc[seg_id, 'q001'] = np.quantile(x,0.001)
 X_test.loc[seg_id, 'ave10'] = stats.trim_mean(x, 0.1 )<save_to_csv>
new_data=new_data.drop('production_companies',axis=1 )
X_test_scaled = scaler.transform(X_test)
 submission['time_to_failure'] = reg1.predict(X_test_scaled)*0.5 + reg2.predict(X_test_scaled)*0.5
submission.to_csv('submission.csv')
 print(os.getcwd() )<set_options>
new_data=new_data.drop('crew',axis=1 )
%matplotlib inline
pd.options.display.precision = 15
warnings.filterwarnings("ignore")
<load_from_csv>
for i in list1:
new_data['title_is'+i]=new_data['title'].fillna('' ).apply(lambda x: 1 if i in x else 0 )
test_seg = pd.read_csv('.. /input/test/seg_004cd2.csv', dtype={'acoustic_data': np.int16, 'time_to_failure': np.float32} )<load_from_csv>
for col in ['tagline', 'overview','title']:
new_data['len_' + col] =new_data[col].fillna('' ).apply(lambda x: len(x))
new_data['words_' + col] = new_data[col].fillna('' ).apply(lambda x: len(x.split(' ')))
 new_data=new_data.drop(col,axis=1 )
%%time
train = pd.read_csv('.. /input/train.csv', dtype={'acoustic_data': np.int16, 'time_to_failure': np.float32} )<define_variables>
new_data['budget_popularity']=new_data['budget']*1.0/new_data['popularity']
chunk_row = 150_000
rows = 150_000
segments = int(np.floor(train.shape[0] / rows))<prepare_x_and_y>
a=new_data.groupby(['release_year'] ).release_month.value_counts()
b=new_data.groupby(['release_year','release_month'] ).release_day.value_counts()
new_data[(new_data['release_year']==2017)&(new_data['release_day']==30)]
X_tr = pd.DataFrame(index=range(segments), dtype=np.float64)
 y_tr = pd.DataFrame(index=range(segments), dtype=np.float64, columns=['time_to_failure'] )<train_on_grid>
new_data['_releaseYear_popularity_ratio'] = new_data['release_year'] / new_data['popularity']
new_data['runtime_to_mean_year'] = new_data['runtime'] / new_data.groupby("release_year")["runtime"].transform('mean')
 new_data['popularity_to_mean_year'] = new_data['popularity'] / new_data.groupby("release_year")["popularity"].transform('mean')
 new_data['budget_to_mean_year'] = new_data['budget'] / new_data.groupby("release_year")["budget"].transform('mean')
def create_features(X, segment_num, chunk):
def add_trend_feature(arr, abs_values=False):
idx = np.array(range(len(arr)))
 if abs_values:
arr = np.abs(arr)
 lr = LinearRegression()
lr.fit(idx.reshape(-1, 1), arr)
 return lr.coef_[0]
percentiles = [10, 50, 75, 90, 99]
moving_avg_window = [1500, 3000, 6000, 9000, 12000]
first_segments = [10000, 50000]
last_segments = [10000, 50000]
rolling_percentiles = [1, 5, 50, 95, 99]
x = pd.Series(chunk['acoustic_data'].values)
 X.loc[segment_num, 'mean'] = x.mean()
X.loc[segment_num, 'std'] = x.std()
X.loc[segment_num, 'var'] = np.var(x)
 X.loc[segment_num, 'max'] = x.max()
X.loc[segment_num, 'min'] = x.min()
z = np.fft.fft(x)
 realFFT = np.real(z)
 imagFFT = np.imag(z)
 X.loc[segment_num, 'Real_mean'] = realFFT.mean()
X.loc[segment_num, 'Real_std'] = realFFT.std()
X.loc[segment_num, 'Imag_mean'] = imagFFT.mean()
X.loc[segment_num, 'Imag_std'] = imagFFT.std()
for f_seg in first_segments:
X.loc[segment_num, 'std_first_{}'.format(f_seg)] = x[:f_seg].std()
X.loc[segment_num, 'mean_first_{}'.format(f_seg)] = x[:f_seg].mean()
X.loc[segment_num, 'Rstd_last_{}'.format(f_seg)] = realFFT[:f_seg].std()
X.loc[segment_num, 'Rmean_last_{}'.format(f_seg)] = realFFT[:f_seg].mean()
for l_seg in last_segments:
X.loc[segment_num, 'std_last_{}'.format(l_seg)] = x[-l_seg:].std()
X.loc[segment_num, 'mean_last_{}'.format(l_seg)] = x[-l_seg:].mean()
X.loc[segment_num, 'Rstd_last_{}'.format(l_seg)] = realFFT[-l_seg:].std()
X.loc[segment_num, 'Rmean_last_{}'.format(l_seg)] = realFFT[-l_seg:].mean()
for percent in percentiles:
X.loc[segment_num, 'percentile_{}'.format(percent)] = np.percentile(x, percent)
 X.loc[segment_num, 'mad'] = x.mad()
X.loc[segment_num, 'kurt'] = x.kurtosis()
X.loc[segment_num, 'skew'] = x.skew()
X.loc[segment_num, 'med'] = x.median()
X.loc[segment_num, 'Hilbert_mean'] = np.abs(hilbert(x)).mean()
X.loc[segment_num, 'Hann_window_mean'] =(convolve(x, hann(150), mode='same')/ sum(hann(150)) ).mean()
X.loc[segment_num, 'trend'] = add_trend_feature(x)
 X.loc[segment_num, 'abs_trend'] = add_trend_feature(x, abs_values=True)
 for mv_avg_window in moving_avg_window:
X.loc[segment_num, 'Moving_average_{}_mean'.format(mv_avg_window)] = x.rolling(window=mv_avg_window ).mean().mean(skipna=True)
 for windows in [10, 100, 1000]:
x_roll_std = x.rolling(windows ).std().dropna().values
x_roll_mean = x.rolling(windows ).mean().dropna().values
X.loc[segment_num, 'ave_roll_std_' + str(windows)] = x_roll_std.mean()
X.loc[segment_num, 'std_roll_std_' + str(windows)] = x_roll_std.std()
X.loc[segment_num, 'max_roll_std_' + str(windows)] = x_roll_std.max()
X.loc[segment_num, 'min_roll_std_' + str(windows)] = x_roll_std.min()
X.loc[segment_num, 'ave_roll_mean_' + str(windows)] = x_roll_mean.mean()
X.loc[segment_num, 'std_roll_mean_' + str(windows)] = x_roll_mean.std()
X.loc[segment_num, 'max_roll_mean_' + str(windows)] = x_roll_mean.max()
X.loc[segment_num, 'min_roll_mean_' + str(windows)] = x_roll_mean.min()
for roll_perc in rolling_percentiles:
X.loc[segment_num, 'p_{}_roll_std_{}'.format(roll_perc, windows)] = np.percentile(x_roll_std, roll_perc)
 X.loc[segment_num, 'p_{}_roll_mean_{}'.format(roll_perc, windows)] = np.percentile(x_roll_mean, roll_perc)
 X.loc[segment_num, 'av_change_abs_roll_std_' + str(windows)] = np.mean(np.diff(x_roll_std))
X.loc[segment_num, 'av_change_rate_roll_std_' + str(windows)] = np.mean(np.nonzero(( np.diff(x_roll_std)/ x_roll_std[:-1])) [0])
 X.loc[segment_num, 'abs_max_roll_std_' + str(windows)] = np.abs(x_roll_std ).max()
X.loc[segment_num, 'av_change_abs_roll_mean_' + str(windows)] = np.mean(np.diff(x_roll_mean))
X.loc[segment_num, 'av_change_rate_roll_mean_' + str(windows)] = np.mean(np.nonzero(( np.diff(x_roll_mean)/ x_roll_mean[:-1])) [0])
 X.loc[segment_num, 'abs_max_roll_mean_' + str(windows)] = np.abs(x_roll_mean ).max()<feature_engineering>
for segment in tqdm_notebook(range(segments)) :
seg = train.iloc[segment*rows:segment*rows+chunk_row]
y = seg['time_to_failure'].values[-1]
y_tr.loc[segment, 'time_to_failure'] = y
create_features(X_tr, segment, seg )<compute_test_metric>
np.abs(X_tr.corrwith(y_tr['time_to_failure'])).sort_values(ascending=False ).head(10 )<feature_engineering>
new_train=new_data.loc[train.index]
new_test=new_data.loc[test.index]
means_dict = {}
for col in X_tr.columns:
if X_tr[col].isnull().any() :
mean_value = X_tr.loc[X_tr[col] != -np.inf, col].mean()
X_tr.loc[X_tr[col] == -np.inf, col] = mean_value
X_tr[col] = X_tr[col].fillna(mean_value)
 means_dict[col] = mean_value<train_model>
x=new_train.drop('revenue',axis=1)
 y=new_train['revenue']
x=x.fillna(0)
 decision_tree = DecisionTreeRegressor(max_depth = 6)
 decision_tree.fit(x, y)
scaler = StandardScaler()
scaler.fit(X_tr)
 X_train_scaled = pd.DataFrame(scaler.transform(X_tr), columns=X_tr.columns )<load_from_csv>
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2, random_state=42)
 model_XG = xgboost.XGBRegressor()
model_XG.fit(x_train,y_train)
 y_predict_rf = model_XG.predict(x_test)
 print(mean_squared_error(y_test, y_predict_rf))
submission = pd.read_csv('.. /input/sample_submission.csv', index_col='seg_id')
 X_test = pd.DataFrame(columns=X_tr.columns, dtype=np.float64, index=submission.index)
 for i, seg_id in enumerate(tqdm_notebook(X_test.index)) :
seg = pd.read_csv('.. /input/test/' + seg_id + '.csv')
 create_features(X_test, seg_id, seg)
 for col in X_test.columns:
if X_test[col].isnull().any() :
X_test.loc[X_test[col] == -np.inf, col] = means_dict[col]
X_test[col] = X_test[col].fillna(means_dict[col])
 X_test_scaled = pd.DataFrame(scaler.transform(X_test), columns=X_test.columns )<choose_model_class>
X_test=new_test.drop('revenue',axis=1)
 X_test=X_test.fillna(0 )
n_fold = 5
folds = KFold(n_splits=n_fold, shuffle=True, random_state=11 )<split>
ypred=model_XG.predict(X_test )
def train_model(X=X_train_scaled, X_test=X_test_scaled, y=y_tr, params=None, folds=folds, model_type='lgb', plot_feature_importance=False, model=None):
oof = np.zeros(len(X))
prediction = np.zeros(len(X_test))
scores = []
feature_importance = pd.DataFrame()
for fold_n,(train_index, valid_index)in enumerate(folds.split(X)) :
print('Fold', fold_n, 'started at', time.ctime())
 X_train, X_valid = X.iloc[train_index], X.iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
if model_type == 'lgb':
model = lgb.LGBMRegressor(**params, n_estimators = 50000, n_jobs = -1)
 model.fit(X_train, y_train,
eval_set=[(X_train, y_train),(X_valid, y_valid)], eval_metric='mae',
verbose=10000, early_stopping_rounds=200)
 y_pred_valid = model.predict(X_valid)
 y_pred = model.predict(X_test, num_iteration=model.best_iteration_)
 if model_type == 'xgb':
train_data = xgb.DMatrix(data=X_train, label=y_train, feature_names=X.columns)
 valid_data = xgb.DMatrix(data=X_valid, label=y_valid, feature_names=X.columns)
 watchlist = [(train_data, 'train'),(valid_data, 'valid_data')]
model = xgb.train(dtrain=train_data, num_boost_round=20000, evals=watchlist, early_stopping_rounds=200, verbose_eval=500, params=params)
 y_pred_valid = model.predict(xgb.DMatrix(X_valid, feature_names=X.columns), ntree_limit=model.best_ntree_limit)
 y_pred = model.predict(xgb.DMatrix(X_test, feature_names=X.columns), ntree_limit=model.best_ntree_limit)
 if model_type == 'sklearn':
model = model
model.fit(X_train, y_train)
 y_pred_valid = model.predict(X_valid ).reshape(-1,)
 score = mean_absolute_error(y_valid, y_pred_valid)
 print(f'Fold {fold_n}.MAE: {score:.4f}.')
 print('')
 y_pred = model.predict(X_test ).reshape(-1,)
 if model_type == 'cat':
model = CatBoostRegressor(iterations=20000, eval_metric='MAE', task_type='GPU', **params)
 model.fit(X_train, y_train, eval_set=(X_valid, y_valid), cat_features=[], use_best_model=True, verbose=False)
 y_pred_valid = model.predict(X_valid)
 y_pred = model.predict(X_test)
 oof[valid_index] = y_pred_valid.reshape(-1,)
 scores.append(mean_absolute_error(y_valid, y_pred_valid))
prediction += y_pred
if model_type == 'lgb':
fold_importance = pd.DataFrame()
fold_importance["feature"] = X.columns
fold_importance["importance"] = model.feature_importances_
fold_importance["fold"] = fold_n + 1
feature_importance = pd.concat([feature_importance, fold_importance], axis=0)
 prediction /= n_fold
print('CV mean score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)))
 if model_type == 'lgb':
feature_importance["importance"] /= n_fold
if plot_feature_importance:
cols = feature_importance[["feature", "importance"]].groupby("feature" ).mean().sort_values(
 by="importance", ascending=False)[:50].index
best_features = feature_importance.loc[feature_importance.feature.isin(cols)]
plt.figure(figsize=(16, 12)) ;
sns.barplot(x="importance", y="feature", data=best_features.sort_values(by="importance", ascending=False)) ;
plt.title('LGB Features(avg over folds)');
return oof, prediction, feature_importance, model
return oof, prediction, model
else:
return oof, prediction, model<init_hyperparams>
dtrain = xgb.DMatrix(x, y)
 dtest = xgb.DMatrix(X_test )
params = {'num_leaves': 128,
'min_data_in_leaf': 79,
'objective': 'huber',
'max_depth': -1,
'learning_rate': 0.01,
"boosting": "gbdt",
"bagging_freq": 5,
"bagging_fraction": 0.8126672064208567,
"bagging_seed": 11,
"metric": 'mae',
"verbosity": -1,
'reg_alpha': 0.1302650970728192,
'reg_lambda': 0.3603427518866501
}
oof_lgb, prediction_lgb, feature_importance, model = train_model(params=params,
model_type='lgb',
plot_feature_importance=True )<compute_test_metric>
xgb_params = {
'eta': 0.08,
'max_depth': 15,
'subsample': 0.7,
'colsample_bytree': 0.7,
'objective': 'reg:linear',
'eval_metric': 'rmse',
'silent': 1
}
cv_output = xgb.cv(xgb_params, dtrain, num_boost_round=1000, early_stopping_rounds=400, verbose_eval=50, show_stdv=False)
 cv_output[['train-rmse-mean', 'test-rmse-mean']].plot()
feature_importance_values = np.abs(X_test_scaled.corrwith(pd.DataFrame(prediction_lgb)[0]))<sort_values>
y_predict1 = abs(model.predict(dtest))
feature_importance_values[feature_importance_values > 0.6].sort_values(ascending=False )<prepare_output>
train=x
y_train=np.log1p(y)
 test=X_test
y=y_train
submission['time_to_failure'] = prediction_lgb<save_to_csv>
n_folds = 5
def rmsle_cv(model):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(train.values)
 rmse = np.sqrt(-cross_val_score(model, train.values,y_train.values, scoring="neg_mean_squared_error", cv=kf))
return(rmse)
 def eval_model(model, name):
start_time = time.time()
score = rmsle_cv(model)
 print("{} score: {:.4f}({:.4f}), execution time: {:.1f}".format(name, score.mean() , score.std() , time.time() -start_time))
submission.to_csv('submission_rolling_v6.csv' )<import_modules>
mod_lasso = make_pipeline(RobustScaler() , Lasso(alpha=0.005, random_state=1))
eval_model(mod_lasso, "lasso")
 mod_enet = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3))
eval_model(mod_enet, "enet")
 mod_cat = CatBoostRegressor(iterations=10000, learning_rate=0.01,
depth=5, eval_metric='RMSE',
colsample_bylevel=0.7, random_seed = 17, silent=True,
bagging_temperature = 0.2, early_stopping_rounds=200)
 eval_model(mod_cat, "cat")
 mod_gboost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
max_depth=5, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state=5)
 eval_model(mod_gboost, "gboost")
 mod_xgb = xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.05, max_depth=10,
min_child_weight=1.7817, n_estimators=2200,
reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213, silent=1,
random_state=7, nthread=-1)
 eval_model(mod_xgb, "xgb")
 mod_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=1023,
learning_rate=0.05, n_estimators=650,
max_bin=58, bagging_fraction=0.80,max_depth=10,
bagging_freq=5, feature_fraction=0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf=7, min_sum_hessian_in_leaf=11)
 eval_model(mod_lgb, "lgb" )
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from tqdm import tqdm
from tqdm import tqdm_notebook
from scipy.signal import hilbert
from scipy.signal import hann
from scipy.signal import convolve<load_from_csv>
class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models, meta_model, n_folds=5):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, X, y):
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=156)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X[train_index], y[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 return self.meta_model_.predict(meta_features)
 mod_stacked = StackingAveragedModels(base_models =(mod_cat, mod_xgb, mod_gboost, mod_lgb), meta_model = mod_lasso)
 eval_model(mod_stacked, "stacked" )
train_data = pd.read_csv('.. /input/train.csv', dtype = {'acoustic_data': np.int16, 'time_to_failure': np.float32} )<prepare_x_and_y>
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
def predict(model):
model.fit(train.values,y_train.values)
 train_pred = model.predict(train.values)
 pred = np.expm1(model.predict(test.values))
print(rmsle(y_train, train_pred))
return(pred )
seg_length = 150000
total_samples = int(np.floor(( train_data.shape[0])/ seg_length))
x_train = pd.DataFrame(index = range(total_samples), dtype = np.float64)
 y_train = pd.DataFrame(index = range(total_samples), columns = ['time_to_failure'], dtype = np.float64 )<feature_engineering>
pre_lasso = predict(mod_lasso)
 pre_enet = predict(mod_enet)
 pre_xgb = predict(mod_xgb)
 pre_gboost = predict(mod_gboost)
 pre_lgb = predict(mod_lgb)
 pre_stack1 = predict(mod_stacked )
<count_missing_values><EOS> 
new_test['id']=new_test.index
new_test['revenue']=pre_stack1
new_test[['id','revenue']].to_csv('submission_Dragon2.csv', index=False)
 new_test[['id','revenue']].head()