Kaggle competition: LANL Earthquake Prediction
<SOS> metric: MAE Kaggle data source: lanl-earthquake-prediction<feature_engineering>
!apt-get remove swig
!apt-get install swig3.0 build-essential -y
!ln -s /usr/bin/swig3.0 /usr/bin/swig
!apt-get install build-essential
!pip install --upgrade setuptools
!pip install auto-sklearn
try:
except:
pass
mean_price = int(train_df['price'].mean())
 test_df.loc[test_df['price']<200,'price'] = mean_price
train_df.loc[train_df['price']<200,'price'] = mean_price<feature_engineering>
!apt-get remove swig
!apt-get install swig3.0 build-essential -y
!ln -s /usr/bin/swig3.0 /usr/bin/swig
!apt-get install build-essential
!pip install --upgrade setuptools
!pip install auto-sklearn
test_df['features'] = test_df['features'].apply(lambda x: list(map(str.strip, map(str.lower, x)))) \ 
.apply(lambda x: [dict_rep_features[i] for i in x if i in dict_rep_features.index])\ 
.apply(lambda x: list(set(x)))
 train_df['features'] = train_df['features'].apply(lambda x: list(map(str.strip, map(str.lower, x)))) \ 
.apply(lambda x: [dict_rep_features[i] for i in x if i in dict_rep_features.index])\ 
.apply(lambda x: list(set(x)))
<prepare_x_and_y>
warnings.filterwarnings("ignore")
 warnings.simplefilter(action='ignore', category=FutureWarning)
def cart2rho(x, y):
rho = np.sqrt(x**2 + y**2)
 return rho
def cart2phi(x, y):
phi = np.arctan2(y, x)
 return phi
def rotation_x(row, alpha):
x = row['latitude']
y = row['longitude']
return x*math.cos(alpha)+ y*math.sin(alpha)
 def rotation_y(row, alpha):
x = row['latitude']
y = row['longitude']
return y*math.cos(alpha)- x*math.sin(alpha)
 def add_rotation(degrees, df):
namex = "rot" + str(degrees)+ "_X"
namey = "rot" + str(degrees)+ "_Y"
df['num_' + namex] = df.apply(lambda row: rotation_x(row, math.pi/(180/degrees)) , axis=1)
 df['num_' + namey] = df.apply(lambda row: rotation_y(row, math.pi/(180/degrees)) , axis=1)
 return df
def operate_on_coordinates(tr_df, te_df):
for df in [tr_df, te_df]:
df["num_rho"] = df.apply(lambda x: cart2rho(x["latitude"] - 40.78222222, x["longitude"]+73.96527777), axis=1)
 df["num_phi"] = df.apply(lambda x: cart2phi(x["latitude"] - 40.78222222, x["longitude"]+73.96527777), axis=1)
 for angle in [15,30,45,60]:
df = add_rotation(angle, df)
 return tr_df, te_df
train_df, test_df = operate_on_coordinates(train_df, test_df)
 features_to_use.extend(['num_rho', 'num_phi', 'num_rot15_X', 'num_rot15_Y', 'num_rot30_X',
'num_rot30_Y', 'num_rot45_X', 'num_rot45_Y', 'num_rot60_X',
'num_rot60_Y'] )<feature_engineering>
train_X_0 = pd.read_csv(".. /input/lanl-master-s-features-creating-0/train_X_features_865.csv")
 train_X_1 = pd.read_csv(".. /input/lanl-master-s-features-creating-1/train_X_features_865.csv")
 y_0 = pd.read_csv(".. /input/lanl-master-s-features-creating-0/train_y.csv", index_col=False, header=None)
 y_1 = pd.read_csv(".. /input/lanl-master-s-features-creating-1/train_y.csv", index_col=False, header=None )
def cap_share(x):
return sum(1 for c in x if c.isupper())/float(len(x)+1)
 for df in [train_df, test_df]:
df['num_cap_share'] = df['description'].apply(cap_share)
 df['num_nr_of_lines'] = df['description'].apply(lambda x: x.count('<br /><br />'))
df['num_redacted'] = 0
df['num_redacted'].ix[df['description'].str.contains('website_redacted')] = 1
df['num_email'] = 0
df['num_email'].ix[df['description'].str.contains('@')] = 1
reg = re.compile(".*?(\(?\d{3}\D{0,3}\d{3}\D{0,3}\d{4} ).*?", re.S)
 def try_and_find_nr(description):
if reg.match(description)is None:
return 0
return 1
df['num_phone_nr'] = df['description'].apply(try_and_find_nr)
 features_to_use.extend(['num_cap_share', 'num_nr_of_lines', 'num_redacted',
'num_email', 'num_phone_nr'] )<feature_engineering>
train_X = pd.concat([train_X_0, train_X_1], axis=0)
 train_X = train_X.reset_index(drop=True)
 print(train_X.shape)
 train_X.head()
train_df["num_photos"] = train_df["photos"].apply(len)
 test_df["num_photos"] = test_df["photos"].apply(len)
 train_df["num_features"] = train_df["features"].apply(len)
 test_df["num_features"] = test_df["features"].apply(len)
 train_df["num_description_words"] = train_df["description"].apply(lambda x: len(x.split(" ")))
 test_df["num_description_words"] = test_df["description"].apply(lambda x: len(x.split(" ")))
 train_df["created"] = pd.to_datetime(train_df["created"])
 test_df["created"] = pd.to_datetime(test_df["created"])
 train_df["created_year"] = train_df["created"].dt.year
test_df["created_year"] = test_df["created"].dt.year
train_df["created_month"] = train_df["created"].dt.month
test_df["created_month"] = test_df["created"].dt.month
train_df["created_day"] = train_df["created"].dt.day
test_df["created_day"] = test_df["created"].dt.day
train_df["created_hour"] = train_df["created"].dt.hour
test_df["created_hour"] = test_df["created"].dt.hour
features_to_use.extend(["num_photos", "num_features", "num_description_words","created_year", "created_month", "created_day", "listing_id", "created_hour"] )<feature_engineering>
train_y = pd.concat([y_0, y_1], axis=0)
 train_y = train_y.reset_index(drop=True)
 train_y.columns = ['time_to_failure']
train_y['time_to_failure'].shape
train_df["price_t"] =train_df["price"]/train_df["bedrooms"]
test_df["price_t"] = test_df["price"]/test_df["bedrooms"]
train_df["room_sum"] = train_df["bedrooms"]+train_df["bathrooms"]
test_df["room_sum"] = test_df["bedrooms"]+test_df["bathrooms"]
features_to_use.extend(["price_t", "room_sum", "num_description_words"] )<define_variables>
test_X = pd.read_csv(".. /input/lanl-master-s-features-creating-0/test_X_features_10.csv")
 test_X.drop('seg_id',inplace=True,axis=1 )
start_values = [0,0,0]
index=list(range(train_df.shape[0]))
random.shuffle(index)
 a=[np.nan]*len(train_df)
 b=[np.nan]*len(train_df)
 c=[np.nan]*len(train_df)
 for i in range(5):
building_level={}
for j in train_df['manager_id'].values:
building_level[j]= start_values.copy()
test_index=index[int(( i*train_df.shape[0])/5):int(((i+1)*train_df.shape[0])/5)]
train_index=list(set(index ).difference(test_index))
for j in train_index:
temp=train_df.iloc[j]
if temp['interest_level']=='low':
building_level[temp['manager_id']][0]+=1
if temp['interest_level']=='medium':
building_level[temp['manager_id']][1]+=1
if temp['interest_level']=='high':
building_level[temp['manager_id']][2]+=1
for j in test_index:
temp=train_df.iloc[j]
if sum(building_level[temp['manager_id']])!=0:
a[j]=building_level[temp['manager_id']][0]*1.0/sum(building_level[temp['manager_id']])
 b[j]=building_level[temp['manager_id']][1]*1.0/sum(building_level[temp['manager_id']])
 c[j]=building_level[temp['manager_id']][2]*1.0/sum(building_level[temp['manager_id']])
 train_df['manager_level_low']=a
train_df['manager_level_medium']=b
train_df['manager_level_high']=c
a=[]
b=[]
c=[]
building_level={}
for j in train_df['manager_id'].values:
building_level[j]= start_values.copy()
for j in range(train_df.shape[0]):
temp=train_df.iloc[j]
if temp['interest_level']=='low':
building_level[temp['manager_id']][0]+=1
if temp['interest_level']=='medium':
building_level[temp['manager_id']][1]+=1
if temp['interest_level']=='high':
building_level[temp['manager_id']][2]+=1
for i in test_df['manager_id'].values:
if i not in building_level.keys() :
a.append(np.nan)
 b.append(np.nan)
 c.append(np.nan)
 else:
a.append(building_level[i][0]*1.0/sum(building_level[i]))
b.append(building_level[i][1]*1.0/sum(building_level[i]))
c.append(building_level[i][2]*1.0/sum(building_level[i]))
test_df['manager_level_low']=a
test_df['manager_level_medium']=b
test_df['manager_level_high']=c
features_to_use.append('manager_level_low')
 features_to_use.append('manager_level_medium')
 features_to_use.append('manager_level_high' )<feature_engineering>
scaler = StandardScaler()
train_columns = train_X.columns
train_X[train_columns] = scaler.fit_transform(train_X[train_columns])
 test_X[train_columns] = scaler.transform(test_X[train_columns] )
train_df["listing_id1"] = train_df["listing_id"] - 68119576.0
test_df["listing_id1"] = test_df["listing_id"] - 68119576.0
train_df["num_price_by_furniture"] =(train_df["price"])/(train_df["bathrooms"] + train_df["bedrooms"] + 1.0)
 test_df["num_price_by_furniture"] =(test_df["price"])/(test_df["bathrooms"] + test_df["bedrooms"] + 1.0)
 train_df["price_latitue"] =(train_df["price"])/(train_df["latitude"]+1.0)
 test_df["price_latitue"] =(test_df["price"])/(test_df["latitude"]+1.0)
 train_df["price_longtitude"] =(train_df["price"])/(train_df["longitude"]-1.0)
 test_df["price_longtitude"] =(test_df["price"])/(test_df["longitude"]-1.0)
 train_df["num_furniture"] = train_df["bathrooms"] + train_df["bedrooms"]
test_df["num_furniture"] = test_df["bathrooms"] + test_df["bedrooms"]
train_df["total_days"] =(train_df["created_month"] -4.0)*30 + train_df["created_day"] + train_df["created_hour"] /25.0
test_df["total_days"] =(test_df["created_month"] -4.0)*30 + test_df["created_day"] + test_df["created_hour"] /25.0
train_df["diff_rank"]= train_df["total_days"]/train_df["listing_id1"]
test_df["diff_rank"]= test_df["total_days"]/test_df["listing_id1"]
features_to_use.extend([ "total_days","diff_rank",
"num_price_by_furniture","price_latitue","price_longtitude",'num_furniture'] )<categorify>
cls = autosklearn.regression.AutoSklearnRegressor(ensemble_memory_limit=5000,ml_memory_limit=10000,time_left_for_this_task=10000,per_run_time_limit=1000,seed=123,resampling_strategy='cv',resampling_strategy_arguments={"folds": 5})
 cls.fit(train_X, train_y,metric=autosklearn.metrics.mean_absolute_error,dataset_name='LANL')
 cls.refit(train_X, train_y )
<count_values><EOS> 
preds = cls.predict(test_X)
 sample_submission = pd.read_csv('.. /input/LANL-Earthquake-Prediction/sample_submission.csv')
 sample_submission['time_to_failure'] = preds.flatten()
sample_submission.to_csv('submission.csv', index=False )