Kaggle competition: Tweet Sentiment Extraction
<SOS> metric: Jaccard Kaggle data source: tweet-sentiment-extraction<predict_on_test>
is_inference_flag = True
try:
tweet_models_dir = os.listdir('/kaggle/input/albert-xl-300/')
 if len(tweet_models_dir)> 0:
is_inference_flag = True
except:
is_inference_flag = False
prediction= model_tuned.predict(X_Ktest )<save_to_csv>
print('Inference flag status :', is_inference_flag )
output=pd.DataFrame({"PassengerId":ids, "Survived":prediction})
 output.to_csv('submission_tuned.csv', index=False )<load_from_csv>
if not is_inference_flag:
!git clone https://github.com/AnandAwasthi/Closed-domain-Question-Answering-fine-tune-Albert
train = pd.read_csv('/kaggle/input/titanic/train.csv')
 test = pd.read_csv('/kaggle/input/titanic/test.csv')
<feature_engineering>
import pandas as pd
import numpy as np
import json
from sklearn.model_selection import train_test_split
datasets = [train,test]
for df in datasets:
df['Title'] = df.Name.str.extract('([A-Za-z]+)\.', expand=False )<feature_engineering>
train_df = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/train.csv')
 test_df = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/test.csv')
 sub_df = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/sample_submission.csv')
for df in datasets:
df['hasCabin'] = np.where(pd.isnull(df['Cabin']),0,1)
 df.loc[pd.isnull(df['Embarked']),'Embarked'] = 'None'
df.drop(['Name','Ticket','Cabin'],axis=1,inplace=True)
 train.head()<categorify>
def jaccard(str1, str2):
a = str1.lower().split()
b = str2.lower().split()
c = set(a ).intersection(set(b))
return float(len(c)) /(len(a)+ len(b)- len(c))
SEED = 1
np.random.seed(SEED)
 le = dict()
le['Sex'] = LabelEncoder()
le['Sex'].fit(train.Sex)
 le['Embarked'] = LabelEncoder()
le['Embarked'].fit(train.Embarked)
 le['Title'] = LabelEncoder()
le['Title'].fit(pd.concat([train.Title, test.Title], axis=0))
for df in datasets:
df['Sex'] = le['Sex'].transform(df['Sex'])
 df['Embarked'] = le['Embarked'].transform(df['Embarked'])
 df['Title'] = le['Title'].transform(df['Title'])
 train.head() 
<feature_engineering>
train_df.dropna(inplace=True )
for df in datasets:
df.loc[pd.isnull(df['Age']), 'Age'] = df['Age'].mean()
for df in datasets:
df.loc[:,'Age'] = np.round(df['Age'] )<feature_engineering>
X_train, X_test = train_test_split(train_df,test_size=0.10, random_state=42 )
for df in datasets:
df.loc[pd.isnull(df['Fare']),'Fare'] = df['Fare'].mean() 
<prepare_x_and_y>
train = np.array(X_train)
 val = np.array(X_test)
 test = np.array(test_df)
 use_cuda = True
x_train0 = train.drop(['PassengerId','Survived'],axis=1)
 y_train0 = train['Survived']
x_test0 = test.drop(['PassengerId'],axis=1)
 sc = StandardScaler()
x_train = sc.fit_transform(x_train0)
 x_test = sc.fit_transform(x_test0)
 y_train = y_train0.values.astype('float32')
<import_modules>
%%time

 def convert_test_qa_json(test):
output = {}
output['version'] = 'v1.0'
output['data'] = []
paragraphs = []
for line in test:
context = line[1]
qas = []
question = line[-1]
qid = line[0]
if type(context)!= str or type(question)!= str:
print(context, type(context))
print(answer, type(answer))
print(question, type(question))
continue
answers = []
answers.append({'answer_start': 1000000, 'text': '__None__'})
 qas.append({'question': question, 'id': qid, 'is_impossible': False, 'answers': answers})
 paragraphs.append({'context': context.lower() , 'qas': qas})
 output['data'].append({'title': 'None', 'paragraphs': paragraphs})
 return output
qa_test = convert_test_qa_json(test)
 with open('data/test.json', 'w')as outfile:
json.dump(qa_test, outfile)
from keras import models
from keras import layers
from keras import optimizers<define_variables>
if not is_inference_flag:
!export SQUAD_DIR=data \
&& python Closed-domain-Question-Answering-fine-tune-Albert/bsie/transformers/examples/run_squad.py \
--model_type albert \
--model_name_or_path albert-base-v2 \
--do_train \
--do_eval \
--do_lower_case \
--train_file $SQUAD_DIR/train.json \
--predict_file $SQUAD_DIR/val.json \
--per_gpu_train_batch_size 12 \
--learning_rate 5e-5 \
--num_train_epochs 1.0 \
--max_seq_length 192 \
--doc_stride 64 \
--output_dir $SQUAD_DIR/models/albert \
--save_steps 100000 \
--threads 4 \
--version_2_with_negative \
--overwrite_output_dir
epochs_num = 100
batch_size = 20
input_dim = len(x_train[0] )<choose_model_class>
AutoConfig,
AutoModelForQuestionAnswering,
AutoTokenizer,
squad_convert_examples_to_features 
)
 if not is_inference_flag:
model_name_or_path = "data/models/"
else:
model_name_or_path = '/kaggle/input/albert-xl-300/'
output_dir = ""
n_best_size = 1
max_answer_length = 254
do_lower_case = True
null_score_diff_threshold = 0.0
def to_list(tensor):
return tensor.detach().cpu().tolist()
config_class, model_class, tokenizer_class =(
 AutoConfig, AutoModelForQuestionAnswering, AutoTokenizer)
 config = config_class.from_pretrained(model_name_or_path)
 tokenizer = tokenizer_class.from_pretrained(
 model_name_or_path, do_lower_case=True)
 model = model_class.from_pretrained(model_name_or_path, config=config)
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 model.to(device)
 processor = SquadV2Processor()
def run_prediction(question_texts, context_text):

 if question_texts[0] != 'neutral':
examples = []
for i, question_text in enumerate(question_texts):
example = SquadExample(
 qas_id=str(i),
question_text=question_text,
context_text=context_text,
answer_text=None,
start_position_character=None,
title="Predict",
is_impossible=False,
answers=None, 
)
 examples.append(example)
 features, dataset = squad_convert_examples_to_features(
 examples=examples,
tokenizer=tokenizer,
max_seq_length=300,
doc_stride=128,
max_query_length=64,
is_training=False,
return_dataset="pt",
threads=1, 
)
 eval_sampler = SequentialSampler(dataset)
 eval_dataloader = DataLoader(dataset, sampler=eval_sampler, batch_size=10)
 all_results = []
for batch in eval_dataloader:
model.eval()
batch = tuple(t.to(device)for t in batch)
 with torch.no_grad() :
inputs = {
"input_ids": batch[0],
"attention_mask": batch[1],
"token_type_ids": batch[2],
}
example_indices = batch[3]
outputs = model(**inputs)
 for i, example_index in enumerate(example_indices):
eval_feature = features[example_index.item() ]
unique_id = int(eval_feature.unique_id)
 output = [to_list(output[i])for output in outputs]
start_logits, end_logits = output
result = SquadResult(unique_id, start_logits, end_logits)
 all_results.append(result)
 output_prediction_file = "predictions.json"
output_nbest_file = "nbest_predictions.json"
output_null_log_odds_file = "null_predictions.json"
predictions = compute_predictions_logits(
 examples,
features,
all_results,
n_best_size,
max_answer_length,
do_lower_case,
output_prediction_file,
output_nbest_file,
output_null_log_odds_file,
False,
True,
null_score_diff_threshold,
tokenizer, 
)
 else:
predictions = OrderedDict([(0, context_text)])
 return predictions
def get_model(input_dim):
model = models.Sequential()
model.add(layers.Dense(units = 7, kernel_initializer = 'lecun_uniform', activation = 'relu', input_dim = input_dim))
model.add(layers.Dense(units = 5, kernel_initializer = 'lecun_uniform', activation = 'relu'))
model.add(layers.Dense(units = 1, kernel_initializer = 'lecun_uniform', activation = 'sigmoid'))
model.compile(optimizer=optimizers.Adam(learning_rate=0.01),
loss='binary_crossentropy', metrics=['accuracy'])
 return model<train_model>
%%time
jaccard_scores = []
predictions_x_test = []
for index, row in X_test.head(10 ).iterrows() :
context = row['text']
selected_text = row['selected_text']
questions = [row['sentiment']]
preds_dict = run_prediction(questions, context)
 for key in preds_dict.keys() :
predicted_text = preds_dict[key]
predictions_x_test.append({'selected_text': selected_text,'predicted_text': predicted_text, 'sentiment': row['sentiment'], 'textID': row['textID']})
 jaccard_score = jaccard(selected_text, predicted_text)
 jaccard_scores.append(jaccard_score)
 print('Jaccard Score', np.mean(jaccard_scores))
predictions_x_test_df = pd.DataFrame.from_dict(predictions_x_test)
 predictions_x_test_df.head()
<predict_on_test><EOS> 
%%time
predictions = []
for index, row in test_df.iterrows() :
context = row['text']
questions = [row['sentiment']]
preds_dict = run_prediction(questions, context)
 for key in preds_dict.keys() :
predicted_text = preds_dict[key]
predictions.append({'textID': row['textID'], 'selected_text': predicted_text})
 predictions_df = pd.DataFrame.from_dict(predictions)
 output_df = sub_df.merge(predictions_df, on ='textID')
 predictions_df.to_csv('submission.csv', index=False)
 predictions_df.head()