Kaggle competition: Предсказание положения космических объектов
<SOS> metric: smape Kaggle data source: sputnik<sort_values>
warnings.filterwarnings('ignore')
 %matplotlib inline
train[['Pclass','Survived']].groupby(['Pclass'], as_index=False ).mean().sort_values(by='Survived',ascending=False )<sort_values>
df = pd.read_csv('.. /input/sputnik/train.csv')
 df['epoch'] = pd.to_datetime(df.epoch)
 df.index = df.epoch
df.drop('epoch', axis = 1, inplace = True)
 df.head()
train[['Sex','Survived']].groupby(['Sex'],as_index=False ).mean().sort_values(by='Survived',ascending=False )<sort_values>
train = df.loc[df['type'] == 'train']
train[['Embarked','Survived']].groupby(['Embarked'],as_index=False ).mean().sort_values(by='Survived',ascending=False )<categorify>
train['error'] = np.linalg.norm(train[['x', 'y', 'z']].values - train[['x_sim', 'y_sim', 'z_sim']].values, axis=1 )
guess_ages = np.zeros(( 3,9))
for dataset in all_data:
dataset['ageFill']=dataset.Age.isnull().map({False:0,True:1})
 med_all = dataset['Age'].median()
for i in range(0,3):
for j in range(0,9):
guess_df=dataset[(dataset['Pclass']==i+1)&\ 
(dataset['SibSp']==j)]['Age'].dropna()
age_guess=guess_df.median()
try:
guess_ages[i,j] = int(age_guess/0.5+0.5)*0.5
except:
guess_ages[i,j]=med_all
for i in range(0,3):
for j in range(0,9):
dataset.loc[(dataset.Age.isnull())&(dataset.Pclass==i+1)&(dataset.SibSp==j),'Age']=guess_ages[i,j]
dataset['Age']=dataset['Age'].astype(int)
 print(guess_ages)
 train.head(10 )<categorify>
test = df.loc[df['type'] == 'test']
freq_port = train.Embarked.dropna().mode() [0]
train.Embarked = train.Embarked.fillna(freq_port)
 print(freq_port)
<feature_engineering>
train_frames = []
for i in range(600):
train_frames.append(train.loc[train['sat_id'] == i] )
test['Fare']=test.Fare.fillna(test.Fare.mean())
 test.info()<feature_engineering>
test_frames = []
for i in range(600):
test_frames.append(test.loc[test['sat_id'] == i] )
for dataset in all_data:
dataset['Title'] = dataset.Name.str.extract('([A-Za-z]+)\.',expand=False)
 pd.crosstab(train['Title'],train['Sex'] )<feature_engineering>
season=24
def cleanTicket(ticket):
ticket = ticket.replace('.' , '')
 ticket = ticket.replace('/' , '')
 ticket = ticket.split()
ticket = map(lambda t : t.strip() , ticket)
 ticket = list(filter(lambda t : not t.isdigit() , ticket))
if len(ticket)> 0:
return ticket[0]
else:
return 'XXX'
for dataset in all_data:
dataset[ 'ticketPos' ] = dataset[ 'Ticket' ].map(cleanTicket)
 train.head()<categorify>
sample_data=train_frames[0].x
for dataset in all_data:
title_mapping = {'Mr':1,'Rare':2,'Master':3,'Miss':4,'Mrs':5}
dataset['Title'] = dataset['Title'].map(title_mapping)
 dataset['Title'] = dataset['Title'].fillna(0)
 train.head()<categorify>
test_stationarity(sample_data )
for dataset in all_data:
dataset['Sex']=dataset.Sex.map({'male':0,'female':1})
 train.head()<categorify>
rcParams['figure.figsize'] = 12, 7
sample_data_diff = sample_data - sample_data.shift(1)
 sample_data_diff.dropna(inplace = True)
 test_stationarity(sample_data_diff, window = 48 )
for dataset in all_data:
dataset['Embarked']=dataset.Embarked.map({'S':0,'Q':1,"C":2})
 train.head()<data_type_conversions>
d=1
D=0
for dataset in all_data:
dataset['AgeBand']=pd.cut(dataset['Age'],5,labels=[0,1,2,3,4])
 dataset['AgeBand']=dataset.AgeBand.astype(int)
 train[['AgeBand','Survived']].groupby(['AgeBand'],as_index=False ).mean().sort_values(by='Survived',ascending=False )<categorify>
from scipy.stats import linregress
for dataset in all_data:
dataset['cabinRec']=dataset.Cabin.isnull().map({False:0,True:1})
 train.head()<categorify>
def prediction(train,test,period = 24):
count = test.shape[0]//period
left = test.shape[0]%period
trend = train.error[-47:].rolling(window=24 ).mean()
os1 =(train.iloc[-period:].error.values ).tolist() *count
if left != 0:
os1 += train.iloc[-period:-(period-left)].error.values.tolist()
seas = np.array(os1)
 a0,*_ = linregress(np.arange(24),trend.dropna().values)
 seas+=np.array(range(seas.shape[0])*a0)
 return seas
for dataset in all_data:
dataset['Fare'].fillna(dataset['Fare'].median() , inplace = True)
 dataset['FareBin'] = pd.qcut(dataset['Fare'], 5,labels=[1,2,3,4,5])
 dataset['FareBin'] = dataset.FareBin.astype(int)
 train.head()<categorify>
pr = prediction(train_frames[0],test_frames[0] )
<drop_column>
y = np.hstack(( np.asarray(train_frames[0].error), pr))
train_df=train.drop(['AgeBand','FareBin','Ticket','Cabin','Name','PassengerId','ticketPos'],axis=1)
 test_df=test.drop(['AgeBand','FareBin','Ticket','Cabin','Name','PassengerId','ticketPos'],axis=1)
 combine_df = [train_df,test_df]
train_df.head()<feature_engineering>
x = np.hstack(( np.asarray(train_frames[0].index),(np.asarray(test_frames[0].index))))
for dataset in combine_df:
dataset['Cabin_Class'] =(dataset.cabinRec+1)*dataset.Pclass<drop_column>
result_errors = []
for i in range(600):
pr_err = prediction(train_frames[i],test_frames[i])
 result_errors.extend(list(pr_err))
sz = len(result_errors )
train_df=train_df.drop('Sex',axis=1)
 test_df=test_df.drop('Sex',axis=1 )<split>
pred_id = list(test.id )
X_train_valid = train_df.drop('Survived',axis=1)
 y_train_valid = train_df['Survived']
X_test = test_df
X_train,X_valid,y_train,y_valid=train_test_split(X_train_valid,y_train_valid,test_size=0.25,random_state=0)
 print(X_train.shape,X_test.shape,X_valid.shape )<train_model>
df_pred_err = pd.DataFrame()
df_pred_err ['id'] = pred_id
df_pred_err['error'] = result_errors
<define_search_space><EOS> 
df_pred_err.to_csv('prediction.csv',sep=",",index = False )