Kaggle competition: LANL Earthquake Prediction
<SOS> metric: MAE Kaggle data source: lanl-earthquake-prediction<feature_engineering>
%matplotlib inline
pd.options.display.precision = 15
warnings.filterwarnings("ignore")
train_df['features'] = train_df["features"].apply(lambda x: " ".join(["_".join(i.split(" ")) for i in x]))
test_df['features'] = test_df["features"].apply(lambda x: " ".join(["_".join(i.split(" ")) for i in x]))
print(train_df["features"].head())
 tfidf = CountVectorizer(stop_words='english', max_features=70)
 te_sparse = tfidf.fit_transform(test_df["features"])
 tr_sparse = tfidf.transform(train_df["features"] )<feature_engineering>
%%time
train = pd.read_csv('.. /input/train.csv', dtype={'acoustic_data': np.int16, 'time_to_failure': np.float32},nrows=6e8 )
tfidfdesc=TfidfVectorizer(min_df=20, max_features=50, strip_accents='unicode',lowercase =True,
analyzer='word', token_pattern=r'\w{16,}', ngram_range=(1, 2), use_idf=False,smooth_idf=False,
sublinear_tf=True, stop_words = 'english')
 train_df['description'] = train_df['description'].apply(lambda x: str(x ).encode('utf-8')if len(x)>2 else "nulldesc")
 test_df['description'] = test_df['description'].apply(lambda x: str(x ).encode('utf-8')if len(x)>2 else "nulldesc")
 te_sparsed = tfidfdesc.fit_transform(test_df["description"])
 tr_sparsed = tfidfdesc.transform(train_df["description"] )<prepare_x_and_y>
min = -100
max = 100
spread = 110
def get_counts(sequence):
counts = [0]*spread
unique_count = np.unique(sequence, return_counts=True)
 for i in range(0,len(unique_count[0])) :
val = unique_count[0][i]
count = unique_count[1][i]
r = count*val
if val <= min:
counts[0] += r
elif val >= max:
counts[-1] += r
else:
counts[int(val/2)+int(spread/2)] += r
return counts
train_X = sparse.hstack([train_df[features_to_use], tr_sparse,tr_sparsed] ).tocsr()
test_X = sparse.hstack([test_df[features_to_use], te_sparse,te_sparsed] ).tocsr()
target_num_map = {'high':0, 'medium':1, 'low':2}
train_y = np.array(train_df['interest_level'].apply(lambda x: target_num_map[x]))
print(train_X.shape, test_X.shape )<save_to_csv>
interval = 75000
counts = [get_counts(train.acoustic_data.values[i:i+150000])for i in tqdm_notebook(range(0,len(train),interval)) ]
ttfs = [train.time_to_failure.values[i] for i in range(0,len(train),interval)]
del train
labels = ["D"+str(i)for i in range(0,len(counts[0])) ]
df = pd.DataFrame(counts, columns=labels)
 ttf_df = pd.DataFrame(ttfs, columns=["expected"])
 df = df.join(ttf_df )
preds, model = runXGB(train_X, train_y, test_X, num_rounds=1000)
 out_df = pd.DataFrame(preds)
 out_df.columns = ["high", "medium", "low"]
out_df["listing_id"] = test_df.listing_id.values
out_df.to_csv("xgb_starter2.csv", index=False )<set_options>
tpath = ".. /input/test"
files = os.listdir(tpath)
 i = 0
test_id = []
test_df = pd.DataFrame(dtype=np.float64, columns=df.columns.values[:-1])
 for f in tqdm_notebook(files):
seg = pd.read_csv(f'{tpath}/{f}')
 converted = get_counts(seg.acoustic_data.values)
 test_df.loc[i] = converted
test_id.append(f.replace(".csv", ""))
i+=1
color = sns.color_palette()
%matplotlib inline<load_from_disk>
num = len(df)
 interval = int(num/100)
 values = int(num/(5*100))
valid_idx = []
for i in range(0,len(df)-values,interval):
for j in range(0,values-1):
valid_idx.append(i+j )
train_df = pd.read_json("/kaggle/input/two-sigma-connect-rental-listing-inquiries/train.json.zip")
 train_df.head()<load_from_disk>
valid_ttfs = np.array([df.iloc[i].expected for i in valid_idx] )
test_df = pd.read_json("/kaggle/input/two-sigma-connect-rental-listing-inquiries/test.json.zip")
 print("Train Rows : ", train_df.shape[0])
 print("Test Rows : ", test_df.shape[0] )<import_modules>
data = TabularDataBunch.from_df(path, df, "expected", valid_idx=valid_idx, test_df=test_df, procs=[Normalize])
from scipy import sparse
import xgboost as xgb
from sklearn import model_selection, preprocessing, ensemble
from sklearn.metrics import log_loss
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer<train_model>
%%time
best_learn = None
best_mae = 9999
for i in range(0, 99):
learn = tabular_learner(data=data, layers=[200,100], metrics=mae, ps=0.5, y_range=(-1,15))
learn.callbacks = [SaveModelCallback(learn, every='improvement', mode='min', name='best')]
learn.fit_one_cycle(20, 1e-2)
 gc.collect()
preds = learn.get_preds(DatasetType.Valid)[0].numpy().flatten()
new_mae = np.abs(valid_ttfs-preds ).mean()
if new_mae < best_mae or not best_learn:
best_learn = learn
best_mae = new_mae
print(f'Run {i} - Best MAE: {best_mae}' )
def runXGB(train_X, train_y, test_X, test_y=None, feature_names=None, seed_val=0, num_rounds=1000):
param = {}
param['objective'] = 'multi:softprob'
param['eta'] = 0.1
param['max_depth'] = 6
param['silent'] = 1
param['num_class'] = 3
param['eval_metric'] = "mlogloss"
param['min_child_weight'] = 1
param['subsample'] = 0.7
param['colsample_bytree'] = 0.7
param['seed'] = seed_val
num_rounds = num_rounds
plst = list(param.items())
 xgtrain = xgb.DMatrix(train_X, label=train_y)
 if test_y is not None:
xgtest = xgb.DMatrix(test_X, label=test_y)
 watchlist = [(xgtrain,'train'),(xgtest, 'test')]
model = xgb.train(plst, xgtrain, num_rounds, watchlist, early_stopping_rounds=20)
 else:
xgtest = xgb.DMatrix(test_X)
 model = xgb.train(plst, xgtrain, num_rounds)
 pred_test_y = model.predict(xgtest)
 return pred_test_y, model<load_from_disk>
preds = best_learn.get_preds(DatasetType.Test)[0].numpy().flatten()
data_path = "/kaggle/input/two-sigma-connect-rental-listing-inquiries/"
train_file = data_path + "train.json.zip"
test_file = data_path + "test.json.zip"
train_df = pd.read_json(train_file)
 test_df = pd.read_json(test_file)
 print(train_df.shape)
 print(test_df.shape )<define_variables>
tpath = ".. /input/test"
files = os.listdir(tpath)
 files = [f.replace(".csv","")for f in files]
files[:3]
features_to_use = ["bathrooms", "bedrooms", "latitude", "longitude", "price"]<feature_engineering>
results = pd.DataFrame({"seg_id":files, "time_to_failure":preds})
 results.head()
<categorify><EOS> 
results.to_csv('submission.csv',index=False )