Kaggle competition: Tweet Sentiment Extraction
<SOS> metric: Jaccard Kaggle data source: tweet-sentiment-extraction<categorify>
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
Actual_1 = 'My life is totally awesome'
Predict_1 = 'awesome life'
Actual_2 = 'We are active kagglers'
Predict_2 = 'We are active kagglers'
print("Jaccard score for first set of scentences: {}".format(jaccard(Actual_1,Predict_1)))
 print("Jaccard score for second set of scentences: {}".format(jaccard(Actual_2,Predict_2)) )
def test_var_sets(train):
cat_cols = [i for i in train.columns if i in categ_cols]
hot = OneHotEncoder(handle_unknown='ignore')
 train_tr = pd.DataFrame(hot.fit_transform(train[cat_cols] ).toarray() , columns=hot.get_feature_names())
 train = pd.concat([train.drop(cat_cols, axis=1), train_tr], axis=1)
 del(train_tr)
 y = train['target']
train = train.drop(['target', 'MobileNo_Avl_Flag'], axis=1)
 X_train, X_valid, y_train, y_valid = train_test_split(train, y, test_size=0.3, random_state = 42)
 lgbt = LGBMClassifier(max_depth=4, learning_rate=0.03, n_estimators=1000, random_state=42)
 lgbt.fit(X_train, y_train)
 y_pred = lgbt.predict_proba(X_valid)[:, 1]
return roc_auc_score(y_valid, y_pred )<define_variables>
stop=set(stopwords.words('english'))
warnings.filterwarnings("ignore")
 cufflinks.go_offline()
cufflinks.set_config_file(world_readable=True, theme='pearl')
var_sets_auc = dict()<feature_engineering>
train=pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv')
 test=pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv')
 train.sample(6 )
for varset in list_of_sets:
trainset = train[list(varset)+ ['target', 'MobileNo_Avl_Flag']]
auc = test_var_sets(trainset)
 var_sets_auc[auc] = varset<sort_values>
def miss_val(df):
total=df.isnull().sum()
return pd.concat([total],axis=1,keys=['Total'])
 print("Missing values for train dataset
")
 print(miss_val(train))
print("---------------------------------------------------------------------")
 print("Missing values for test dataset
")
 print(miss_val(test))
best_set = sorted(var_sets_auc.items() , key=lambda x: -x[0])[0][1]<features_selection>
train=train.dropna()
train.shape
train_best = train[best_set]
test_best = test[best_set]
cat_cols = [i for i in train_best.columns if i in categ_cols]<count_unique_values>
train['target']=train['selected_text'].str.lower()
train_best.nunique()<categorify>
def find_link(string):
url = re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F])) +', string)
 return "".join(url)
 train['target_url']=train['target'].apply(lambda x: find_link(x))
df=pd.DataFrame(train.loc[train['target_url']!=""]['sentiment'].value_counts() ).reset_index()
df.rename(columns={"index": "sentiment", "sentiment": "url_count"} )
hot = OneHotEncoder(handle_unknown='ignore')
 train_tr = pd.DataFrame(hot.fit_transform(train_best[cat_cols] ).toarray() , columns=hot.get_feature_names())
 test_tr = pd.DataFrame(hot.transform(test_best[cat_cols] ).toarray() , columns=hot.get_feature_names())
 train_best = pd.concat([train_best.drop(cat_cols, axis=1), train_tr], axis=1)
 test_best = pd.concat([test_best.drop(cat_cols, axis=1), test_tr], axis=1)
 del(train_tr, test_tr )<prepare_x_and_y>
def find_punct(text):
line = re.findall(r'[!"\$%&'() *+,\-.\/:;=
string="".join(line)
 return list(string )
y = train['target'] 
<split>
train['target_punct']=train['target'].apply(lambda x:find_punct(x))
train['target_punct_len']=train['target'].apply(lambda x:len(find_punct(x)) )
X_train, X_valid, y_train, y_valid = train_test_split(train_best, y, test_size=0.3, random_state = 42 )<train_on_grid>
punc_df=pd.DataFrame(train,columns=['target_punct','sentiment'])
 punc_df=punc_df[punc_df['target_punct'].map(lambda d: len(d)) > 0]
punc_df=punc_df.explode('target_punct')
 positive_df=pd.DataFrame(punc_df.loc[punc_df['sentiment']=="positive"]['target_punct'].value_counts() ).reset_index().rename(columns={'index': 'punct','target_punct':'pos_punct'})
 negative_df=pd.DataFrame(punc_df.loc[punc_df['sentiment']=="negative"]['target_punct'].value_counts() ).reset_index().rename(columns={'index': 'punct','target_punct':'neg_punct'})
 neutral_df=pd.DataFrame(punc_df.loc[punc_df['sentiment']=="neutral"]['target_punct'].value_counts() ).reset_index().rename(columns={'index': 'punct','target_punct':'neut_punct'})
lgbm_params = {'max_depth': [3,5,7],
'learning_rate':[0.05, 0.01, 0.03],
'n_estimators':[1000, 1200, 1400]}
lgbt = LGBMClassifier(random_state=42)
 cv = KFold(n_splits=5, shuffle=True, random_state=42)
 lgbm_grid = GridSearchCV(lgbt, lgbm_params, cv=cv, scoring='roc_auc', verbose=1, n_jobs=-1)
 lgbm_grid.fit(X_train, y_train )<find_best_params>
train['star']=train['target'].apply(lambda x:find_star(x))
train.loc[train['star']!=0]['sentiment'].value_counts().to_frame()
lgbm_grid.best_params_<compute_train_metric>
def find_only_star(text):
if len(text.split())==1:
line=re.findall(r'[*]{2,5}',text)
 return len(line)
 else:
return 0
lgbt = LGBMClassifier(max_depth=3, learning_rate=0.05, n_estimators=1000, random_state=42)
 lgbt.fit(X_train, y_train)
 y_pred = lgbt.predict_proba(X_valid)[:, 1]
y_pred =(y_pred - np.min(y_pred)) /(np.max(y_pred)- np.min(y_pred))
roc_auc_score(y_valid, y_pred )<train_on_grid>
train['only_star']=train['target'].apply(lambda x:find_only_star(x))
train.loc[train['only_star']==1]['sentiment'].value_counts()
y = train['target']
train = train.drop(['target', 'MobileNo_Avl_Flag'], axis=1)
 test = test.drop(['MobileNo_Avl_Flag'], axis=1)
 X_train, X_valid, y_train, y_valid = train_test_split(train, y, test_size=0.3, random_state = 42)
 lgbm_params = {'max_depth': [4],
'learning_rate':np.arange(0.03, 0.06, 0.01),
'n_estimators':[500, 1000, 1500, 2000]}
lgbt = LGBMClassifier(random_state=42)
 cv = KFold(n_splits=5, shuffle=True, random_state=42)
 lgbm_grid = GridSearchCV(lgbt, lgbm_params, cv=cv, scoring='roc_auc')
 lgbm_grid.fit(X_train, y_train )<find_best_model_class>
train['target']= np.where(train['only_star']==1,"abusive",train['target'] )
lgbt = LGBMClassifier(max_depth=4, learning_rate=0.03, n_estimators=1000)
 lgbt.fit(X_train, y_train)
 y_pred = lgbt.predict_proba(X_valid)[:, 1]
roc_auc_score(y_valid, y_pred )<train_model>
def remove_link(string):
text = re.sub('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F])) +'," ",string)
 return " ".join(text.split() )
lgbt = LGBMClassifier(max_depth=4, learning_rate=0.03, n_estimators=1000)
 lgbt.fit(train, y )<predict_on_test>
def remove_punct(text):
line = re.sub(r'[!"\$%&'() *+,\-.\/:;=
return " ".join(line.split() )
answer_1 = lgbt.predict_proba(test)[:, 1]
answer_1<prepare_output>
train['target']=train['target'].apply(lambda x:remove_link(x))
train['target']=train['target'].apply(lambda x:remove_punct(x))
answer1 = pd.DataFrame(columns=['ID', 'Predicted'])
 answer1['ID'] = test.index
answer1['Predicted'] = answer_1<save_to_csv>
def create_corpus(data,feature,sentiment):
corpus=[]
for x in data[data['sentiment']==sentiment][feature].str.split() :
for i in x:
corpus.append(i)
 return corpus
answer1.to_csv('answer228.csv', index=None )<load_from_csv>
def corpus_sentiment(data,feature,sentiment):
corpus=create_corpus(data,feature,sentiment)
 dic=defaultdict(int)
 for word in corpus:
if word not in stop:
dic[word]+=1
top=sorted(dic.items() , key=lambda x:x[1],reverse=True)
 x,y=zip(*top)
 return x,y
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
df_train = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-3/train.csv')
 df_test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-3/test.csv')
<data_type_conversions>
def count_values(df,feature):
total=df.loc[:,feature].value_counts(dropna=False)
 percent=round(df.loc[:,feature].value_counts(dropna=False,normalize=True)*100,2)
 return pd.concat([total,percent],axis=1,keys=['Total','Percent'])
def fillState(state, country):
if state == "NA": return country
return state
def fixData(input_set):
input_set['Province_State'].fillna("NA", inplace=True)
 input_set['Province_State'] = input_set.loc[:, ['Province_State', 'Country_Region']].apply(lambda x : fillState(x['Province_State'], x['Country_Region']), axis=1)
 input_set['Date'] = pd.to_datetime(input_set['Date'], infer_datetime_format=True)
 input_set.loc[:, 'Date'] = input_set.Date.dt.strftime("%m%d")
 input_set["Date"] = input_set["Date"].astype(int)
 return input_set 
<prepare_x_and_y>
count_values(train,'sentiment' )
X_train = df_train
X_test = df_test
X_train = fixData(X_train)
 X_test = fixData(X_test)
 X_train.head()<categorify>
count_values(train,'sentiment' )
<categorify>
full_data=pd.concat([train,test])
 full_data['text']=full_data['text'].str.lower()
full_data.shape
label_encoder = preprocessing.LabelEncoder()
countries = X_test.Country_Region.unique()<prepare_x_and_y>
full_data['text_punct']=full_data['text'].apply(lambda x:find_punct(x))
full_data['text_punct_len']=full_data['text'].apply(lambda x:len(find_punct(x)) )
sub = pd.DataFrame({'ForecastId': [], 'ConfirmedCases': [], 'Fatalities': []})
 sub = []
for country in countries:
province_list = X_train.loc[X_train['Country_Region'] == country].Province_State.unique()
for province in province_list:
X_train2 = X_train.loc[(X_train['Country_Region'] == country)&(X_train['Province_State'] == province),['Date']].astype('int')
 Y_train21 = X_train.loc[(X_train['Country_Region'] == country)&(X_train['Province_State'] == province),['ConfirmedCases']]
Y_train22 = X_train.loc[(X_train['Country_Region'] == country)&(X_train['Province_State'] == province),['Fatalities']]
X_test2 = X_test.loc[(X_test['Country_Region'] == country)&(X_test['Province_State'] == province), ['Date']].astype('int')
 X_forecastId2 = X_test.loc[(X_test['Country_Region'] == country)&(X_test['Province_State'] == province), ['ForecastId']]
X_forecastId2 = X_forecastId2.values.tolist()
X_forecastId2 = [v[0] for v in X_forecastId2]
model2 = XGBRegressor(n_estimators=1020)
 model2.fit(X_train2, Y_train21)
 Y_pred2 = model2.predict(X_test2)
 model3 = XGBRegressor(n_estimators=1020)
 model3.fit(X_train2, Y_train22)
 Y_pred3 = model3.predict(X_test2)
 for j in range(len(Y_pred2)) :
dic = { 'ForecastId': X_forecastId2[j], 'ConfirmedCases': Y_pred2[j], 'Fatalities': Y_pred3[j]}
sub.append(dic)
<save_to_csv>
punc_text_df=pd.DataFrame(full_data,columns=['text_punct','sentiment'])
 punc_text_df=punc_text_df[punc_text_df['text_punct'].map(lambda d: len(d)) > 0]
punc_text_df=punc_text_df.explode('text_punct')
 positive_text_df=pd.DataFrame(punc_text_df.loc[punc_text_df['sentiment']=="positive"]['text_punct'].value_counts() ).reset_index().rename(columns={'index': 'punct','text_punct':'pos_punct'})
 negative_text_df=pd.DataFrame(punc_text_df.loc[punc_text_df['sentiment']=="negative"]['text_punct'].value_counts() ).reset_index().rename(columns={'index': 'punct','text_punct':'neg_punct'})
 neutral_text_df=pd.DataFrame(punc_text_df.loc[punc_text_df['sentiment']=="neutral"]['text_punct'].value_counts() ).reset_index().rename(columns={'index': 'punct','text_punct':'neut_punct'})
submission = pd.DataFrame(sub)
 submission[['ForecastId','ConfirmedCases','Fatalities']].to_csv(path_or_buf='submission.csv',index=False )<import_modules>
full_data['text']=full_data['text'].apply(lambda x:remove_link(x))
full_data['text']=full_data['text'].apply(lambda x:remove_punct(x))
<train_model>
full_data.loc[full_data['text']=="",['text']]="nothing"
<install_modules>
def corpus_sentiment_stop(data,feature,sentiment):
corpus=create_corpus(data,feature,sentiment)
 dic=defaultdict(int)
 for word in corpus:
if word in stop:
dic[word]+=1
top=sorted(dic.items() , key=lambda x:x[1],reverse=True)
 x,y=zip(*top)
 return x,y
!pip install pyramid-arima<set_options>
train_word=full_data[:train.shape[0]]
test_word=full_data[train.shape[0]:]
warnings.filterwarnings("ignore")
 test = pd.read_csv(".. /input/covid19-global-forecasting-week-2/test.csv")
 train = pd.read_csv(".. /input/covid19-global-forecasting-week-2/train.csv")
 train.head()<count_unique_values>
pos_text_list=list(corpus_sentiment(train_word,'text','positive')[0])
 pos_target_list=list(corpus_sentiment(train_word,'target','positive')[0])
 neg_text_list=list(corpus_sentiment(train_word,'text','negative')[0])
 neg_target_list=list(corpus_sentiment(train_word,'target','negative')[0])
 neutral_text_list=list(corpus_sentiment(train_word,'text','neutral')[0])
 neutral_target_list=list(corpus_sentiment(train_word,'target','neutral')[0])
 pos_common_words_list=list(set(pos_text_list ).intersection(pos_target_list))
neg_common_words_list=list(set(neg_text_list ).intersection(neg_target_list))
neutral_common_words_list=list(set(neutral_text_list ).intersection(neutral_target_list))
no_countr = train['Country_Region'].nunique()
no_province = train['Province_State'].nunique()
no_countr_with_prov = len(train[train['Province_State'].isna() ==False]['Country_Region'].unique())
 total_forecasting_number = no_province + no_countr - no_countr_with_prov+2
no_days = train['Date'].nunique()
print('there are ', no_countr, 'unique Countries/Regionions, each with ', no_days, 'days of data, all of them having the same dates.There are also ',no_province, 'Provinces/States which can be found on ', no_countr_with_prov, 'countries/ regions.' )<prepare_x_and_y>
results_jaccard=[]
for ind,row in train_word.iterrows() :
sentence1 = row.text
sentence2 = row.target
jaccard_score = jaccard(sentence1,sentence2)
 results_jaccard.append([sentence1,sentence2,jaccard_score] )
df = confirmed_total_date.copy()
df = pd.DataFrame({'date': [df.index[i] for i in range(len(df)) ] , 'cases': df['ConfirmedCases'].values.reshape(1,-1)[0].tolist() })
 dfog = df.copy()
def l_regr(x,y):
model = LinearRegression().fit(x, y)
 return model
x = df['cases']
x = x.drop(x.index[-1] ).values.reshape(( -1, 1))
y = df['cases']
y = y.drop(y.index[0])
 ex_slope = l_regr(x,y ).coef_
d = 0
for i in range(1,5):
plt.plot(df['cases'])
 plt.show()
plt.close()
df['prev_cases'] = df['cases'].shift(1)
 df['cases'] =(df['cases'] - df['prev_cases'])
 df = df.drop(['prev_cases'],axis=1)
 df = df.drop(df.index[0])
 x = df['cases']
x = x.drop(y.index[-1] ).values.reshape(( -1, 1))
y = df['cases']
y = y.drop(y.index[0])
 model = l_regr(x,y)
 if(abs(model.coef_)> ex_slope):
print('this is it! ', ex_slope)
 break
d += 1
ex_slope = model.coef_
print(model.coef_ )<create_dataframe>
jaccard = pd.DataFrame(results_jaccard,columns=["text","target","jaccard_score"])
 train_word = train_word.merge(jaccard,how='outer' )
index = 1
cases_pred= []
fatalities_pred = []
pbar = tqdm(total=total_forecasting_number)
 while index < total_forecasting_number+3:
x = train['ConfirmedCases'].iloc[[i for i in range(no_days*(index-1),no_days*index)]].values
z = train['Fatalities'].iloc[[i for i in range(no_days*(index-1),no_days*index)]].values
index += 1
no_nul_cases = pd.DataFrame(x)
 no_nul_cases = no_nul_cases[no_nul_cases.values != 0]
if(not no_nul_cases.empty):
X = [xi for xi in no_nul_cases.values]
try:
model = pyramid.arima.auto_arima(X,seasonal=True, m=12)
 pred = model.predict(31)
 pred = pred.astype(int)
 pred = pred.tolist()
except:
model = l_regr(np.array([i for i in range(len(X)) ] ).reshape(-1, 1),X)
 pred = [(model.coef_*(len(X)+i)+ model.intercept_ ).astype('int')[0][0] for i in range(1,32)]
else:
pred = [0] * 31
pred = x[-12:].astype(int ).tolist() + pred
cases_pred+=pred
no_nul_fatalities = pd.DataFrame(z)
 no_nul_fatalities = no_nul_fatalities[no_nul_fatalities.values != 0]
if(not no_nul_fatalities.empty):
Z = [zi for zi in no_nul_fatalities.values]
try:
model = pyramid.arima.auto_arima(Z, seasonal=False, m=12)
 pred = model.predict(31)
 pred = pred.astype(int)
 pred = pred.tolist()
except:
model = l_regr(np.array([i for i in range(len(Z)) ] ).reshape(-1, 1),Z)
 pred = [(model.coef_*(len(Z)+i)+ model.intercept_ ).astype('int')[0][0] for i in range(1,32)]
else:
pred = [0] * 31
pred = z[-12:].astype(int ).tolist() + pred
fatalities_pred+=pred
pbar.update(1)
 pbar.close()<save_to_csv>
print("Jaccard score equals 1 : {}%".format(round(train_word.loc[train_word['jaccard_score']==1].shape[0]/train_word.shape[0]*100,2)))
 print("Jaccard score less than 0.3 : {}%".format(round(train_word.loc[train_word['jaccard_score']<0.3].shape[0]/train_word.shape[0]*100,2)) )
submission = pd.DataFrame({'ForecastId': [i for i in range(1,len(cases_pred)+1)] ,'ConfirmedCases': cases_pred, 'Fatalities': fatalities_pred})
 filename = 'submission.csv'
submission.to_csv(filename,index=False )<set_options>
train_word['difference']=abs(train_word['text_tweet_length']-train_word['target_tweet_length'] )
plt.style.use('ggplot')
 %matplotlib inline 
<load_from_csv>
MAX_LEN = 96
PATH = '.. /input/tf-roberta/'
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=PATH+'vocab-roberta-base.json',
merges_file=PATH+'merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True 
)
 sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}
df_train = pd.read_csv('.. /input/homework-for-students4plus/train.csv',index_col=0)
 df_test = pd.read_csv('.. /input/homework-for-students4plus/test.csv',index_col=0 )<data_type_conversions>
train = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv' ).fillna('')
 ct=train.shape[0]
input_ids=np.ones(( ct,MAX_LEN),dtype="int32")
 attention_mask=np.zeros(( ct,MAX_LEN),dtype="int32")
 token_type_ids=np.zeros(( ct,MAX_LEN),dtype="int32")
 start_tokens=np.zeros(( ct,MAX_LEN),dtype="int32")
 end_tokens=np.zeros(( ct,MAX_LEN),dtype="int32" )
df_train['issue_d'] = df_train['issue_d'].astype(str)
 df_train = df_train[df_train['issue_d'].str.contains("2015")]<prepare_x_and_y>
for k in range(train.shape[0]):
text1 = " "+" ".join(train.loc[k,'text'].split())
 text2 = " ".join(train.loc[k,'selected_text'].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ': chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []
idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0:
toks.append(i)
 s_tok = sentiment_id[train.loc[k,'sentiment']]
input_ids[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask[k,:len(enc.ids)+5] = 1
if len(toks)>0:
start_tokens[k,toks[0]+1] = 1
end_tokens[k,toks[-1]+1] = 1
y_train = df_train.loan_condition
X_train = df_train.drop(['loan_condition'], axis=1)
 X_test = df_test<load_from_csv>
test = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv' ).fillna('')
 ct_test = test.shape[0]
input_ids_t = np.ones(( ct_test,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct_test,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct_test,MAX_LEN),dtype='int32')
 for k in range(test.shape[0]):
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test.loc[k,'sentiment']]
input_ids_t[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask_t[k,:len(enc.ids)+5] = 1
gdp = pd.read_csv('.. /input/homework-for-students4plus/US_GDP_by_State.csv')
 gdp = gdp.rename(columns={'State': 'city'})
 state_long = pd.read_csv('.. /input/homework-for-students4plus/statelatlong.csv' )<define_variables>
def build_model() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 config = RobertaConfig.from_pretrained(PATH+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'pretrained-roberta-base.h5',config=config)
 x = bert_model(ids,attention_mask=att,token_type_ids=tok)
 x1 = tf.keras.layers.Dropout(0.1 )(x[0])
 x1 = tf.keras.layers.Conv1D(128, 2,padding='same' )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Conv1D(64, 2,padding='same' )(x1)
 x1 = tf.keras.layers.Dense(1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.1 )(x[0])
 x2 = tf.keras.layers.Conv1D(128, 2, padding='same' )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Conv1D(64, 2, padding='same' )(x2)
 x2 = tf.keras.layers.Dense(1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 model.compile(loss='binary_crossentropy', optimizer=optimizer)
 return model
gdp_train = gdp.query('year == 2014')
 gdp_test = gdp.query('year == 2015' )<merge>
start_time=dt.now()
n_splits=5
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
DISPLAY=1
for i in range(5):
print('
print('
print('
K.clear_session()
model = build_model()
model.load_weights('.. /input/model4/v4-roberta-%i.h5'%i)
 print('Predicting Test...')
 preds = model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/n_splits
preds_end += preds[1]/n_splits
end_time=dt.now()
print(" ")
 print(" ")
 print("Time Taken to run above code :",(end_time-start_time ).total_seconds() /60," minutes" )
merge_train = pd.merge(gdp_train[['city','State & Local Spending','Gross State Product','Real State Growth %','Population(million)']],
state_long[['State','City']], left_on='city', right_on='City', how='inner' ).drop(['City'],axis=1 ).drop(['city'],axis=1)
 merge_test = pd.merge(gdp_test[['city','State & Local Spending','Gross State Product','Real State Growth %','Population(million)']],
state_long[['State','City']], left_on='city', right_on='City', how='inner' ).drop(['City'],axis=1 ).drop(['city'],axis=1 )<feature_engineering>
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 if a>b:
st = test.loc[k,'text']
else:
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 all.append(st )
<prepare_x_and_y><EOS> 
test['selected_text'] = all
submission=test[['textID','selected_text']]
submission.to_csv('submission.csv',index=False)
 submission.head(5 )