Kaggle competition: New York City Taxi Fare Prediction
<SOS> metric: RMSE Kaggle data source: new-york-city-taxi-fare-prediction<define_variables>
plt.style.use('ggplot' )
<import_modules>
train = pd.read_csv('.. /input/train.csv', nrows = 5000000 )
print('TF version',tf.__version__ )<define_variables>
train = train[train.fare_amount > 0]
MAX_LEN = 96
PATH = '.. /input/tf-roberta/'
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=PATH+'vocab-roberta-base.json',
merges_file=PATH+'merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True 
)
 sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}
train = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv' ).fillna('')
 train.head()<define_variables>
test = pd.read_csv('.. /input/test.csv')
 test['longitude_diff'] = test['dropoff_longitude'] - test['pickup_longitude']
test['latitude_diff'] = test['dropoff_latitude'] - test['pickup_latitude']
ct = train.shape[0]
input_ids = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids = np.zeros(( ct,MAX_LEN),dtype='int32')
 start_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 end_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(train.shape[0]):
text1 = " "+" ".join(train.loc[k,'text'].split())
 text2 = " ".join(train.loc[k,'selected_text'].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ': chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []; idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0: toks.append(i)
 s_tok = sentiment_id[train.loc[k,'sentiment']]
input_ids[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask[k,:len(enc.ids)+5] = 1
if len(toks)>0:
start_tokens[k,toks[0]+1] = 1
end_tokens[k,toks[-1]+1] = 1<feature_engineering>
train = train.loc[train.fare_amount < 35]
test = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv' ).fillna('')
 ct = test.shape[0]
input_ids_t = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(test.shape[0]):
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test.loc[k,'sentiment']]
input_ids_t[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask_t[k,:len(enc.ids)+5] = 1<load_pretrained>
train = train.loc[train.pickup_longitude < -72.5]
train = train.loc[train.pickup_longitude > -74.5]
def build_model() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 config = RobertaConfig.from_pretrained(PATH+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'pretrained-roberta-base.h5',config=config)
 x = bert_model(ids,attention_mask=att,token_type_ids=tok)
 x1 = tf.keras.layers.Dropout(0.1 )(x[0])
 x1 = tf.keras.layers.Conv1D(1,1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.1 )(x[0])
 x2 = tf.keras.layers.Conv1D(1,1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 model.compile(loss='categorical_crossentropy', optimizer=optimizer)
 return model<string_transform>
train = train.loc[train.pickup_latitude < 42]
train = train.loc[train.pickup_latitude > 40]
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 if(len(a)==0)&(len(b)==0): return 0.5
c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))<define_variables>
train = train.loc[train.dropoff_longitude < -72.5]
train = train.loc[train.dropoff_longitude > -74.5]
<predict_on_test>
train = train.loc[train.dropoff_latitude < 42]
train = train.loc[train.dropoff_latitude > 40]
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
DISPLAY=1
n_splits = 5
for i in range(5):
print('
print('
print('
K.clear_session()
model = build_model()
model.load_weights('.. /input/tf-roberta-weights/v0-roberta-%i.h5'%i)
 print('Predicting Test...')
 preds = model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/n_splits
preds_end += preds[1]/n_splits<string_transform>
train['longitude_diff'] = train['dropoff_longitude'] - train['pickup_longitude']
train['latitude_diff'] = train['dropoff_latitude'] - train['pickup_latitude']
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 if a>b:
st = test.loc[k,'text']
else:
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 all.append(st )<save_to_csv>
train = train.loc[train.longitude_diff > -0.9]
train = train.loc[train.longitude_diff < 0.5]
train = train.loc[train.latitude_diff > -0.7]
train = train.loc[train.latitude_diff < 0.3]
test['selected_text'] = all
test[['textID','selected_text']].to_csv('submission.csv',index=False)
 pd.set_option('max_colwidth', 60)
 test.sample(25 )<init_hyperparams>
train = train.loc[train.passenger_count > 0]
train = train.loc[train.passenger_count <= 6]
tf.random.set_seed(1234)
 class SentenceSelector(BaseEstimator):
def __init__(self, maxlen=96, bert_path="/kaggle/input/tf-roberta", epochs=3, batch_size=32, learning_rate=3e-5,
optimizer="adam", validation_split=0.2, ignore_neutrals=True, verbose=False):
self.verbose = verbose
self.maxlen = maxlen
self.bert_path = bert_path
self.bert_vocab = self.bert_path + "/vocab-roberta-base.json"
self.bert_merges = self.bert_path + "/merges-roberta-base.txt"
self.bert_config = self.bert_path + "/config-roberta-base.json"
self.bert_pretrained = self.bert_path + "/pretrained-roberta-base.h5"
self.tokenizer = tokenizers.ByteLevelBPETokenizer(vocab_file=self.bert_vocab,merges_file=self.bert_merges,
lowercase=True, add_prefix_space=True)
 self.model = None
self.epochs = epochs
self.batch_size = batch_size
self.learning_rate = learning_rate
self.optimizer = optimizer
self.validation_split = validation_split
self.ignore_neutrals = ignore_neutrals
self.sentiment_col = "sentiment"
self.text_col = "text"
self.sentiment_dict = {"positive": self.tokenizer.encode("positive" ).ids[0],
"negative": self.tokenizer.encode("negative" ).ids[0],
"neutral": self.tokenizer.encode("neutral" ).ids[0]}
def fit(self, X, y):
if self.ignore_neutrals == True:
X_temp = X.to_dict(orient="records")
 X_new = []
y_new = []
for i in range(len(X_temp)) :
if X_temp[i][self.sentiment_col] != "neutral":
X_new.append(X_temp[i])
 y_new.append(y[i])
 X = pd.DataFrame(X_new)
 y = pd.Series(y_new)
 if isinstance(X, pd.DataFrame):
X = X.values
if isinstance(y, pd.DataFrame)or isinstance(y, pd.Series):
y = y.values
input_ids, attention_mask, token_type_ids, start_tokens, end_tokens = self._preprocessing_train(X, y)
 K.clear_session()
model = self._build_model()
model.fit(x=[input_ids, attention_mask, token_type_ids], y=[start_tokens, end_tokens],
epochs=self.epochs, batch_size=self.batch_size, verbose=1, validation_split=self.validation_split)
 self.model = model
def _build_model(self):
ids = tf.keras.layers.Input(( self.maxlen,), dtype=tf.int32)
 att = tf.keras.layers.Input(( self.maxlen,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( self.maxlen,), dtype=tf.int32)
 if self.optimizer.lower() == "rmsprop":
optimizer = tf.keras.optimizers.RMSprop(learning_rate=self.learning_rate)
 else:
optimizer = tf.keras.optimizers.Adam(learning_rate=self.learning_rate)
 config = transformers.RobertaConfig.from_pretrained(self.bert_config)
 bert_model = transformers.TFRobertaModel.from_pretrained(self.bert_pretrained, config=config)
 x = bert_model(ids,attention_mask=att,token_type_ids=tok)
 x1 = tf.keras.layers.Dropout(0.1 )(x[0])
 x1 = tf.keras.layers.Conv1D(128, 2,padding='same' )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Conv1D(64, 2,padding='same' )(x1)
 x1 = tf.keras.layers.Dense(1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.1 )(x[0])
 x2 = tf.keras.layers.Conv1D(128, 2, padding='same' )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Conv1D(64, 2, padding='same' )(x2)
 x2 = tf.keras.layers.Dense(1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 model.compile(loss="binary_crossentropy", optimizer=optimizer)
 return model
def _preprocessing_train(self, X, y):
input_ids = np.ones(( X.shape[0], self.maxlen), dtype="int32")
 attention_mask = np.zeros(( X.shape[0], self.maxlen), dtype="int32")
 token_type_ids = np.zeros(( X.shape[0], self.maxlen), dtype="int32")
 start_tokens = np.zeros(( X.shape[0], self.maxlen), dtype="int32")
 end_tokens = np.zeros(( X.shape[0], self.maxlen), dtype="int32")
 for k in range(X.shape[0]):
text1 = " " + " ".join(X[k][0].split())
 text2 = " ".join(y[k].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx + len(text2)] = 1
if text1[idx-1] == ' ':
chars[idx-1] = 1
enc = self.tokenizer.encode(text1)
 offsets = []; idx = 0
for t in enc.ids:
w = self.tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0: toks.append(i)
 s_tok = self.sentiment_dict[X[k][1]]
input_ids[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask[k,:len(enc.ids)+5] = 1
if len(toks)>0:
start_tokens[k,toks[0]+1] = 1
end_tokens[k,toks[-1]+1] = 1
return input_ids, attention_mask, token_type_ids, start_tokens, end_tokens
def _preprocessing_test(self, X):
input_ids = np.ones(( X.shape[0], self.maxlen),dtype="int32")
 attention_mask = np.zeros(( X.shape[0], self.maxlen),dtype="int32")
 token_type_ids = np.zeros(( X.shape[0], self.maxlen),dtype="int32")
 for k in range(X.shape[0]):
text1 = " "+" ".join(X[k][0].split())
 enc = self.tokenizer.encode(text1)
 s_tok = self.sentiment_dict[X[k][1]]
input_ids[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask[k,:len(enc.ids)+5] = 1
return input_ids, attention_mask, token_type_ids
def score(self, X_test, y_test):
if self.model == None:
raise AttributeError('The model is not trained yet.')
 if self.ignore_neutrals == True:
preds = self._predict_ignore_neutrals(X_test)
 else:
preds = self.predict(X_test)
 scores = []
for pred, true in zip(preds, y_test):
score = self.jaccard(pred, true)
 scores.append(score)
 return np.mean(scores)
 def jaccard(self, str1, str2):
"The same score function of the competition with only an added of try except to avoid error when we have empty strings."
a = set(str1.lower().split())
 b = set(str2.lower().split())
 c = a.intersection(b)
 try:
jaccard_score = float(len(c)) /(len(a)+ len(b)- len(c))
except Exception as e:
print("An error occurred when compute jaccard score:", e)
 jaccard_score = float(0)
 finally:
return jaccard_score
def predict(self, X):
if self.model == None:
raise AttributeError('The model is not trained yet.')
 if self.ignore_neutrals == True:
return self._predict_ignore_neutrals(X)
 if isinstance(X, pd.DataFrame):
X = X.values
input_ids, attention_mask, token_type_ids = self._preprocessing_test(X)
 preds = self.model.predict([input_ids, attention_mask, token_type_ids])
 selected_text = []
for k in range(X.shape[0]):
start = np.argmax(preds[0][k,])
 end = np.argmax(preds[1][k,])
 if start > end:
text = X[k][0]
else:
text = " "+" ".join(X[k][0].split())
 enc = self.tokenizer.encode(text)
 text = self.tokenizer.decode(enc.ids[start-1:end])
 selected_text.append(text)
 return selected_text
def _predict_ignore_neutrals(self, X):
if self.model == None:
raise AttributeError('The model is not trained yet.')
 X.reset_index(drop=True, inplace=True)
 selected_text = []
for each in range(len(X)) :
if X[self.sentiment_col][each] == "neutral":
selected_text.append(X[self.text_col][each])
 else:
X_temp = X.loc[each]
X_temp = X_temp[[self.text_col, self.sentiment_col]]
X_temp = np.array([X_temp.values])
 input_ids, attention_mask, token_type_ids = self._preprocessing_test(X_temp)
 preds = self.model.predict([input_ids, attention_mask, token_type_ids])
 for k in range(X_temp.shape[0]):
start = np.argmax(preds[0][k,])
 end = np.argmax(preds[1][k,])
 if start > end:
text = X_temp[k][0]
else:
text = " "+" ".join(X_temp[k][0].split())
 enc = self.tokenizer.encode(text)
 text = self.tokenizer.decode(enc.ids[start-1:end])
 selected_text.append(text)
 return selected_text
def save_model(self, filepath):
if self.model == None:
raise AttributeError('The model is not trained yet.')
 tf.keras.models.save_model(self.model, filepath)
 def load_model(self, filepath, force=False):
if self.model != None and force == False:
raise AttributeError('You already have a loaded model.If you want to reload inform the parameter force=True')
 self.model = tf.keras.models.load_model(filepath)
 def GridSearch(param_grid, X_train, y_train, X_test, y_test, train_size=None, test_size=None, log_scores=False, verbose=False):
if isinstance(train_size, float):
x1, x2, y1, y2 = train_test_split(X_train, y_train, train_size=train_size, random_state=42, stratify=X_train["sentiment"])
 if verbose:
print("train dataset reduced from {} to {} rows".format(len(X_train), len(x1)))
 X_train = x1.copy()
y_train = y1.copy()
if isinstance(test_size, float):
x1, x2, y1, y2 = train_test_split(X_test, y_test, test_size=test_size, random_state=42, stratify=X_test["sentiment"])
 if verbose:
print("test dataset reduced from {} to {} rows".format(len(X_test), len(x2)))
 X_test = x2.copy()
y_test = y2.copy()
param_grid = list(ParameterGrid(param_grid))
if verbose:
print("The param_grid will result in {} fits.".format(len(param_grid)))
 scores = []
best_score = 0
best_param = None
exec_counter = 0
for params in param_grid:
exec_counter += 1
if verbose:
print("{}/{}".format(exec_counter, len(param_grid)))
 model = SentenceSelector()
model.set_params(**params)
 model.fit(X_train, y_train)
 score = model.score(X_test, y_test)
 scores.append({"score": score, "params": model.get_params() })
 for each in scores:
if each["score"] > best_score:
best_score = each["score"]
best_param = each["params"]
if log_scores:
with open('gridsearch_scores_log.json', 'w')as outfile:
json.dump(scores, outfile)
 result = {"score": best_score, "params": best_param}
if verbose:
print(result)
 return result
def input_files(data_map, tpu=False):
if tpu:
kaggle = KaggleDatasets()
for dataset in data_map.keys() :
GCS_DS_PATH = KaggleDatasets().get_gcs_path(dataset)
 if isinstance(data_map.get(dataset), str):
data_map[dataset] = GCS_DS_PATH
if isinstance(data_map.get(dataset), dict):
for file in data_map[dataset].keys() :
data_map[dataset][file] = GCS_DS_PATH + "/" + data_map[dataset][file]
else:
for dataset in data_map.keys() :
if isinstance(data_map.get(dataset), str):
data_map[dataset] = "/kaggle/input/{}".format(data_map[dataset])
 if isinstance(data_map.get(dataset), dict):
for file in data_map[dataset].keys() :
data_map[dataset][file] = "/kaggle/input/{}/{}".format(dataset, data_map[dataset][file])
 return data_map<set_options>
train['year'] = train.pickup_datetime.apply(lambda x: x[:4])
 test['year'] = test.pickup_datetime.apply(lambda x: x[:4] )
AUTO = tf.data.experimental.AUTOTUNE
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tpu_on = True
!pip install gcsfs
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
tpu_on = False
if tpu_on:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()<load_from_csv>
train['month'] = train.pickup_datetime.apply(lambda x: x[5:7])
 test['month'] = test.pickup_datetime.apply(lambda x: x[5:7] )
data_map = {
"tweet-sentiment-extraction": {
"train": "train.csv",
"test": "test.csv",
"submission": "sample_submission.csv"
},
"tf-roberta": "tf-roberta/"
}
files = input_files(data_map, tpu=tpu_on)
 files<load_from_csv>
train['hour'] = train.pickup_datetime.apply(lambda x: x[11:13])
 test['hour'] = test.pickup_datetime.apply(lambda x: x[11:13] )
df_train = pd.read_csv(files["tweet-sentiment-extraction"]["train"])
 df_train.fillna('', inplace=True)
 print(df_train.shape)
 df_train.head()<prepare_x_and_y>
train['pickup_datetime'] = train.pickup_datetime.apply(
 lambda x: datetime.datetime.strptime(x[:10], '%Y-%m-%d'))
test['pickup_datetime'] = test.pickup_datetime.apply(
 lambda x: datetime.datetime.strptime(x[:10], '%Y-%m-%d'))
X = df_train.drop(columns=["textID", "selected_text"])
 y = df_train["selected_text"].values<split>
train['day_of_week'] = train.pickup_datetime.apply(lambda x: x.weekday())
 test['day_of_week'] = test.pickup_datetime.apply(lambda x: x.weekday() )
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.30, random_state=42, stratify=X["sentiment"])
 print("train:", len(X_train))
print("test:", len(X_test))<init_hyperparams>
train['pickup_date'] = train.pickup_datetime.apply(lambda x: x.date())
 test['pickup_date'] = test.pickup_datetime.apply(lambda x: x.date() )
parameters = {
"maxlen": 141,
"epochs": 3,
"batch_size": 32,
"learning_rate": 3e-5,
"validation_split": 0.2,
"optimizer": "adam",
"ignore_neutrals": False,
"bert_path": files["tf-roberta"],
"model_filepath": "model.sav",
"GridSearch": False,
"ParameterGrid": {
"maxlen": [96, 141],
"epochs": [3, 4, 5],
"batch_size": [16, 32],
"learning_rate": [2e-5, 3e-5, 5e-5],
"ignore_neutrals": [True, False],
"optimizer": ["adam"],
"verbose": [True],
"validation_split": [0.2],
"bert_path": [files["tf-roberta"]]
}
}<train_on_grid>
cal = USFederalHolidayCalendar()
holidays = cal.holidays(start='2009-01-01', end='2015-12-31' ).to_pydatetime()
train['holidat_or_not'] = train.pickup_datetime.apply(lambda x: 1 if x in holidays else 0)
 test['holidat_or_not'] = test.pickup_datetime.apply(lambda x: 1 if x in holidays else 0 )
if parameters["GridSearch"]:
gridsearch = GridSearch(parameters["ParameterGrid"], X_train, y_train, X_test, y_test, train_size=0.3, test_size=0.1, log_scores=True, verbose=True)
 model = SentenceSelector()
model.set_params(**gridsearch["params"])
 else:
model = SentenceSelector(maxlen=parameters["maxlen"], epochs=parameters["epochs"], batch_size=parameters["batch_size"],
ignore_neutrals=parameters["ignore_neutrals"], learning_rate=parameters["learning_rate"],
optimizer=parameters["optimizer"], validation_split=parameters["validation_split"])
 model.fit(X_train, y_train)
 model.save_model(parameters["model_filepath"] )<find_best_params>
train = train.drop(['key','pickup_datetime','pickup_date'],axis=1)
 test = test.drop(['key','pickup_datetime','pickup_date'],axis=1 )
model.get_params()<sort_values>
train['year'] = train['year'].astype('int')
 train['hour'] = train['hour'].astype('int')
 test['year'] = test['year'].astype('int')
 test['hour'] = test['hour'].astype('int' )
test_val.sort_values(by="score", ascending=True ).head(20 )<sort_values>
from sklearn.preprocessing import StandardScaler
test_val.sort_values(by="score", ascending=False ).head(20 )<load_from_csv>
sample = train[:50000][['pickup_longitude','pickup_latitude']]
df_test = pd.read_csv(files["tweet-sentiment-extraction"]["test"])
 df_test.fillna('', inplace=True)
 print(df_test.shape)
 df_test.head()<predict_on_test>
scaler = StandardScaler()
sample = scaler.fit_transform(sample )
df_test["selected_text"] = model.predict(df_test[["text", "sentiment"]] )<load_from_csv>
sample = train[:50000][['dropoff_longitude','dropoff_latitude']]
df_sub = pd.read_csv(files["tweet-sentiment-extraction"]["submission"])
 df_sub.fillna('', inplace=True)
 print(df_sub.shape)
 df_sub.head()<merge>
model = KMeans(n_clusters = 5)
 model.fit(train[['pickup_longitude','pickup_latitude']])
 pickup_labels = model.predict(train[['pickup_longitude','pickup_latitude']] )
df_sub = df_sub.merge(df_test, on="textID")
 df_sub.drop(columns=["selected_text_x", "text", "sentiment"], errors="ignore", inplace=True)
 df_sub.columns = ["textID", "selected_text"]
df_sub<save_to_csv>
train['pickup_cluster'] = pickup_labels
pickup_cluster = pd.get_dummies(train['pickup_cluster'],prefix='pickup_cluster',drop_first=True)
 train = pd.concat([train,pickup_cluster],axis=1 ).drop('pickup_cluster',axis=1 )
df_sub.to_csv("submission.csv", index=False )<save_to_csv>
model2 = KMeans(n_clusters = 8)
 model2.fit(train[['dropoff_longitude','dropoff_latitude']])
 dropoff_labels = model2.predict(train[['dropoff_longitude','dropoff_latitude']] )
predictions = json.load(open('/kaggle/working/predictions_.json', 'r'))
submission = pd.read_csv(open('/kaggle/input/tweet-sentiment-extraction/sample_submission.csv', 'r'))
for i in range(len(submission)) :
id_ = submission['textID'][i]
if test_df['sentiment'][i] == 'neutral':
submission.loc[i, 'selected_text'] = test_df['text'][i]
else:
submission.loc[i, 'selected_text'] = predictions[id_]
submission.to_csv('submission.csv', index=False )<feature_engineering>
train['dropoff_cluster'] = dropoff_labels
dropoff_cluster = pd.get_dummies(train['dropoff_cluster'],prefix='dropoff_cluster',drop_first=True)
 train = pd.concat([train,dropoff_cluster],axis=1 ).drop('dropoff_cluster',axis=1 )
def find_all(input_str, search_str):
l1 = []
length = len(input_str)
 index = 0
while index < length:
i = input_str.find(search_str, index)
 if i == -1:
return l1
l1.append(i)
 index = i + 1
return l1
def do_qa_train(train):
output = {}
output['version'] = 'v1.0'
output['data'] = []
paragraphs = []
for line in train:
context = line[1]
qas = []
question = line[-1]
qid = line[0]
answers = []
answer = line[2]
if type(answer)!= str or type(context)!= str or type(question)!= str:
print(context, type(context))
print(answer, type(answer))
print(question, type(question))
continue
answer_starts = find_all(context, answer)
 for answer_start in answer_starts:
answers.append({'answer_start': answer_start, 'text': answer.lower() })
 break
qas.append({'question': question, 'id': qid, 'is_impossible': False, 'answers': answers})
 paragraphs.append({'context': context.lower() , 'qas': qas})
 output['data'].append({'title': 'None', 'paragraphs': paragraphs})
 return paragraphs
def do_qa_test(test):
paragraphs = []
for line in test:
context = line[1]
qas = []
question = line[-1]
qid = line[0]
if type(context)!= str or type(question)!= str:
print(context, type(context))
print(answer, type(answer))
print(question, type(question))
continue
answers = []
answers.append({'answer_start': 1000000, 'text': '__None__'})
 qas.append({'question': question, 'id': qid, 'is_impossible': False, 'answers': answers})
 paragraphs.append({'context': context.lower() , 'qas': qas})
 output['data'].append({'title': 'None', 'paragraphs': paragraphs})
 return paragraphs
use_cuda = False
train_df = read_train()
test_df = read_test()
submission_df_distil = read_submission()
train = np.array(train_df)
 test = np.array(test_df)
 qa_train = do_qa_train(train)
 with open('data/train.json', 'w')as outfile:
json.dump(qa_train, outfile)
 output = {}
output['version'] = 'v1.0'
output['data'] = []
qa_test = do_qa_test(test)
 with open('data/test.json', 'w')as outfile:
json.dump(qa_test, outfile)
<categorify>
train['pickup_cluster'] = pickup_labels
train['dropoff_cluster'] = dropoff_labels
def post_process(submission_df, test_df):
index_to_selected_text = {}
for i, row in test_df.iterrows() :
_id = row[0]
text = row[1]
sentiment = row[2]
if len(text.split(" ")) <= 3 or sentiment == "neutral":
index_to_selected_text[_id] = text
submission_rows = submission_df.to_dict("records")
 new_rows = []
for row in submission_rows:
_id = row['textID']
if _id in index_to_selected_text:
new_row = deepcopy(row)
 new_row['selected_text'] = index_to_selected_text[_id]
else:
new_row = row
new_rows.append(new_row)
 return pd.DataFrame(new_rows)
<choose_model_class>
train.groupby(['pickup_cluster','dropoff_cluster'])['fare_amount'].agg(['mean','std', 'count'] )
MODEL_PATH = '/kaggle/input/transformers-pretrained-distilbert/distilbert-base-uncased-distilled-squad/'
model = QuestionAnsweringModel('distilbert',
MODEL_PATH,
args={"reprocess_input_data": True,
"overwrite_output_dir": True,
"learning_rate": 8e-05,
"num_train_epochs": 3,
"max_seq_length": 192,
"weight_decay": 0.001,
"doc_stride": 64,
"save_model_every_epoch": False,
"fp16": False,
"do_lower_case": True,
'max_query_length': 8,
'max_answer_length': 150
},
use_cuda=True)
 model.train_model('data/train.json')
 predictions = model.predict(qa_test)
 predictions_df = pd.DataFrame.from_dict(predictions)
 submission_df_distil['selected_text'] = predictions_df['answer']
submission_df_distil = post_process(submission_df_distil, test_df)
 submission_df_distil.to_csv('data/submission.csv', index=False)
 submission_df_distil.to_csv('submission.csv', index=False)
<import_modules>
pickup_test_labels = model.predict(test[['pickup_longitude','pickup_latitude']])
 test['pickup_cluster'] = pickup_test_labels
pickup_cluster = pd.get_dummies(test['pickup_cluster'],prefix='pickup_cluster',drop_first=True)
 test = pd.concat([test,pickup_cluster],axis=1 ).drop('pickup_cluster',axis=1 )
print('TF version',tf.__version__ )<load_from_csv>
dropoff_test_labels = model2.predict(test[['dropoff_longitude','dropoff_latitude']])
 test['dropoff_cluster'] = dropoff_test_labels
dropoff_cluster = pd.get_dummies(test['dropoff_cluster'],prefix='dropoff_cluster',drop_first=True)
 test = pd.concat([test,dropoff_cluster],axis=1 ).drop('dropoff_cluster',axis=1 )
def read_train() :
train=pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv')
 train['text']=train['text'].astype(str)
 train['selected_text']=train['selected_text'].astype(str)
 return train
def read_test() :
test=pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv')
 test['text']=test['text'].astype(str)
 return test
def read_submission() :
test=pd.read_csv('.. /input/tweet-sentiment-extraction/sample_submission.csv')
 return test
train_df = read_train()
test_df = read_test()
submission_df = read_submission()<string_transform>
train = pd.get_dummies(train,columns=['month','hour','day_of_week'],drop_first=True)
 test = pd.get_dummies(test,columns=['month','hour','day_of_week'],drop_first=True )
def jaccard(str1, str2):
a = set(str(str1 ).lower().split())
 b = set(str(str2 ).lower().split())
 c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))<define_variables>
from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import train_test_split
from sklearn import metrics
MAX_LEN = 96
PATH = '.. /input/tf-roberta/'
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=PATH+'vocab-roberta-base.json',
merges_file=PATH+'merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True 
)
 sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}<define_variables>
train = train.drop(['pickup_cluster','dropoff_cluster'],axis=1 )
ct = train_df.shape[0]
input_ids = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids = np.zeros(( ct,MAX_LEN),dtype='int32')
 start_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 end_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(train_df.shape[0]):
text1 = " "+" ".join(train_df.loc[k,'text'].split())
 text2 = " ".join(train_df.loc[k,'selected_text'].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ': chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []; idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0: toks.append(i)
 s_tok = sentiment_id[train_df.loc[k,'sentiment']]
input_ids[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask[k,:len(enc.ids)+5] = 1
if len(toks)>0:
start_tokens[k,toks[0]+1] = 1
end_tokens[k,toks[-1]+1] = 1<categorify>
X = train.drop('fare_amount',axis=1)
 y = train[['fare_amount']]
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42 )
ct = test_df.shape[0]
input_ids_t = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(test_df.shape[0]):
text1 = " "+" ".join(test_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test_df.loc[k,'sentiment']]
input_ids_t[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask_t[k,:len(enc.ids)+5] = 1<load_pretrained>
clf = RandomForestRegressor(n_estimators=30,min_samples_split=5,min_samples_leaf=3,random_state=21)
 clf.fit(X_train,y_train )
def build_model() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 config = RobertaConfig.from_pretrained(PATH+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'pretrained-roberta-base.h5',config=config)
 x = bert_model(ids,attention_mask=att,token_type_ids=tok)
 x1 = tf.keras.layers.Dropout(0.1 )(x[0])
 x1 = tf.keras.layers.Conv1D(128, 2,padding='same' )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Conv1D(64, 2,padding='same' )(x1)
 x1 = tf.keras.layers.Dense(1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.1 )(x[0])
 x2 = tf.keras.layers.Conv1D(128, 2, padding='same' )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Conv1D(64, 2, padding='same' )(x2)
 x2 = tf.keras.layers.Dense(1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 model.compile(loss='binary_crossentropy', optimizer=optimizer)
 return model<define_variables>
predictions = clf.predict(X_test)
 error = np.sqrt(metrics.mean_squared_error(y_test,predictions))
print(error )
<choose_model_class><EOS> 
predictions = clf.predict(test)
 submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['fare_amount'] = predictions
submission.to_csv('submission.csv',index=False )