Kaggle competition: PetFinder.my Adoption Prediction
<SOS> metric: QuadraticWeightedKappa Kaggle data source: petfinder.my-adoption-prediction<feature_engineering>
print("Python version: {}".format(sys.version))
print("pandas version: {}".format(pd.__version__))
print("matplotlib version: {}".format(matplotlib.__version__))
print("NumPy version: {}".format(np.__version__))
print("SciPy version: {}".format(sp.__version__))
print("IPython version: {}".format(IPython.__version__))
print("scikit-learn version: {}".format(sklearn.__version__))
warnings.filterwarnings('ignore')
 warnings.simplefilter(action="ignore", category=ConvergenceWarning)
 print('-'*25)
 print(check_output(["ls", ".. /input"] ).decode("utf8"))
start4,end4=run_fold(3 )<define_variables>
%matplotlib inline
mpl.style.use('ggplot')
 sns.set_style('darkgrid')
 pylab.rcParams['figure.figsize'] = 12,8
start =(start1+start2+start3+start4)/4
end =(end1+end2+end3+end4)/4<define_variables>
MLA = [
ensemble.AdaBoostClassifier() ,
ensemble.BaggingClassifier() ,
ensemble.ExtraTreesClassifier() ,
ensemble.GradientBoostingClassifier() ,
ensemble.RandomForestClassifier() ,
linear_model.LogisticRegressionCV() ,
linear_model.PassiveAggressiveClassifier() ,
linear_model.RidgeClassifierCV() ,
linear_model.SGDClassifier() ,
linear_model.Perceptron() ,
naive_bayes.BernoulliNB() ,
naive_bayes.GaussianNB() ,
neighbors.KNeighborsClassifier() ,
tree.DecisionTreeClassifier() ,
tree.ExtraTreeClassifier() ,
discriminant_analysis.LinearDiscriminantAnalysis() ,
discriminant_analysis.QuadraticDiscriminantAnalysis() ,
XGBClassifier()
]
def train_model(data, MLA_list = MLA, print_feature_table=False, print_feature_score=False, top_n=8):
target = data['AdoptionSpeed']
X_train = data.drop(['AdoptionSpeed'],axis=1)
 MLA_columns = ['MLA Name', 'MLA Parameters','MLA cohen_kappa_score','MLA Time']
MLA_compare = pd.DataFrame(columns = MLA_columns)
 MLA_predict = data['AdoptionSpeed']
scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
 row_index = 0
for alg in MLA_list:
MLA_name = alg.__class__.__name__
MLA_compare.loc[row_index, 'MLA Name'] = MLA_name
MLA_compare.loc[row_index, 'MLA Parameters'] = str(alg.get_params())
 kf = StratifiedKFold(n_splits=5, random_state=42, shuffle=True)
 kappa_score = make_scorer(cohen_kappa_score, weights='quadratic')
 cv_results = model_selection.cross_validate(alg, X_train, target, cv = kf, scoring=kappa_score)
 MLA_compare.loc[row_index, 'MLA Time'] = cv_results['fit_time'].mean()
MLA_compare.loc[row_index, 'MLA cohen_kappa_score'] = cv_results['test_score'].mean()
row_index+=1
MLA_compare.sort_values(by = ['MLA cohen_kappa_score'], ascending = False, inplace = True)
 sns.barplot(x='MLA cohen_kappa_score', y = 'MLA Name', data = MLA_compare, color = 'b')
 plt.title('Machine Learning Algorithm Accuracy Score
')
 plt.xlabel('Accuracy Score(%)')
 plt.ylabel('Algorithm')
 if print_feature_table:
for alg in MLA_list:
alg.fit(X_train, target)
 if hasattr(alg, 'feature_importances_'):
feat_imp = pd.DataFrame({'importance':alg.feature_importances_})
 feat_imp['feature'] = X_train.columns
feat_imp.sort_values(by='importance', ascending=False, inplace=True)
 feat_imp = feat_imp.iloc[:top_n]
feat_imp.sort_values(by='importance', inplace=True)
 feat_imp = feat_imp.set_index('feature', drop=True)
 feat_imp.plot.barh(title=alg.__class__.__name__)
 plt.xlabel('Feature Importance Score')
 plt.show()
if print_feature_score:
print("Top {} features in descending order of importance".format(top_n))
display(feat_imp.sort_values(by='importance', ascending=False))
return MLA_compare
for x,y in dset:
print(y)
 break<load_from_csv>
train = pd.read_csv('.. /input/train/train.csv')
 test = pd.read_csv('.. /input/test/test.csv')
 train['dataset_type'] = 'train'
test['dataset_type'] = 'test'
test = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/test.csv')
 test['selected_text']=test['text'].apply(lambda x:x)
 dset = TweetDataset(tweet = test.text.values,
sentiment = test.sentiment.values,
selected_text = test.selected_text.values,
MAX_LEN=MAX_LEN,
TOKENIZER=config.TOKENIZER)
 final_output=[]
i=0
for x,y in dset:
ids = x["ids"]
token_type_ids = x["token_type_ids"]
mask =x["mask"]
sentiment = y["sentiment"]
orig_selected = y["orig_selected"]
orig_tweet = y["orig_tweet"]
targets_start = y["targets_start"]
targets_end = y["targets_end"]
offsets = y["tweet_offsets"].numpy()
selected_tweet = orig_selected
tweet_sentiment = sentiment
_, output_sentence = calculate_jaccard_score(
 original_tweet=orig_tweet,
target_string=selected_tweet,
sentiment_val=tweet_sentiment,
idx_start=np.argmax(start[i]),
idx_end=np.argmax(end[i]),
offsets=offsets 
)
 final_output.append(output_sentence)
 i+=1 
<feature_engineering>
useColumns = ['Type', 'Age', 'Breed1', 'Breed2', 'Gender', 'Color1', 'Color2',
'Color3', 'MaturitySize', 'FurLength', 'Vaccinated', 'Dewormed',
'Sterilized', 'Health', 'Quantity', 'Fee', 'State',
'VideoAmt', 'PhotoAmt', 'AdoptionSpeed']
test['out']=final_output<save_to_csv>
train['Name'] = train['Name'].fillna('Unnamed')
 test['Name'] = test['Name'].fillna('Unnamed')
 train['has_name'] = train['Name'].apply(lambda x: 0 if x == 'Unnamed' else 1)
 test['has_name'] = test['Name'].apply(lambda x: 0 if x == 'Unnamed' else 1 )
def post_process(selected):
return " ".join(set(selected.lower().split()))
sample = pd.read_csv(".. /input/tweet-sentiment-extraction/sample_submission.csv")
 sample.loc[:, 'selected_text'] = final_output
sample.selected_text = sample.selected_text.map(post_process)
 sample.to_csv("submission.csv", index=False )<import_modules>
train['has_name'] = train['Name'].apply(lambda x: 0 if x == 'No Name' or x == 'Unnamed' else 1)
 test['has_name'] = test['Name'].apply(lambda x: 0 if x == 'No Name' or x == 'Unnamed' else 1 )
import numpy as np
import pandas as pd
import json<load_from_csv>
top_3 = [
ensemble.AdaBoostClassifier() ,
ensemble.GradientBoostingClassifier() ,
XGBClassifier()
]
useColumns = ['Type', 'Age', 'Breed1', 'Breed2', 'Gender', 'Color1', 'Color2',
'Color3', 'MaturitySize', 'FurLength', 'Vaccinated', 'Dewormed',
'Sterilized', 'Health', 'Quantity', 'Fee', 'State',
'VideoAmt', 'PhotoAmt', 'AdoptionSpeed', 'has_name']
pd_train = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/train.csv')
 pd_test = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/test.csv' )<prepare_x_and_y>
def top_names(df, top_percent):
df_withnames = df[df.has_name != 0]
items = df_withnames.shape[0]
top_names = []
counter = 0
for i,v in df_withnames.Name.value_counts().items() :
if(counter/items)>top_percent:
break
top_names.append(i)
 counter = counter + v
return top_names
train = np.array(pd_train)
 test = np.array(pd_test )<find_best_params>
topnames = top_names(train.append(test), 0.2)
 train['has_topname'] = train['Name'].apply(lambda row: 1 if row in topnames else 0)
 test['has_topname'] = test['Name'].apply(lambda row: 1 if row in topnames else 0 )
def find_all(input_str, search_str):
l1 = []
length = len(input_str)
 index = 0
while index < length:
i = input_str.find(search_str, index)
 if i == -1:
return l1
l1.append(i)
 index = i + 1
return l1<define_variables>
cut_points = [1,2,3,4,8,12,24,36,48]
labels = ["1m","2m","3m","4m","8m","1y","2y","3y","4y","5y and over"]
train["age_bin"] = binning(train["Age"], cut_points, labels)
 labels = [1,2,3,4,5,6,7,8,9,10]
train["age_bin"] = binning(train["Age"], cut_points, labels)
 test["age_bin"] = binning(test["Age"], cut_points, labels)
 label = LabelEncoder()
train['age_bin'] = label.fit_transform(train['age_bin'])
 test['age_bin'] = label.fit_transform(test['age_bin'] )
output = {}
output['version'] = 'v1.0'
output['data'] = []
for line in train:
paragraphs = []
context = line[1]
qas = []
question = line[-1]
qid = line[0]
answers = []
answer = line[2]
if type(answer)!= str or type(context)!= str or type(question)!= str:
print(context, type(context))
print(answer, type(answer))
print(question, type(question))
continue
answer_starts = find_all(context, answer)
 for answer_start in answer_starts:
answers.append({'answer_start': answer_start, 'text': answer})
 qas.append({'question': question, 'id': qid, 'is_impossible': False, 'answers': answers})
 paragraphs.append({'context': context, 'qas': qas})
 output['data'].append({'title': 'None', 'paragraphs': paragraphs})
 with open('data/train.json', 'w')as outfile:
json.dump(output, outfile )<categorify>
top_3 = [
ensemble.AdaBoostClassifier() ,
ensemble.GradientBoostingClassifier() ,
XGBClassifier()
]
useColumns = ['Type', 'Breed1', 'Breed2', 'Gender', 'Color1', 'Color2',
'Color3', 'MaturitySize', 'FurLength', 'Vaccinated', 'Dewormed',
'Sterilized', 'Health', 'Quantity', 'Fee', 'State',
'VideoAmt', 'PhotoAmt', 'AdoptionSpeed', 'has_name', 'has_topname', 'age_bin']
output = {}
output['version'] = 'v1.0'
output['data'] = []
for line in test:
paragraphs = []
context = line[1]
qas = []
question = line[-1]
qid = line[0]
if type(context)!= str or type(question)!= str:
print(context, type(context))
print(answer, type(answer))
print(question, type(question))
continue
answers = []
answers.append({'answer_start': 1000000, 'text': '__None__'})
 qas.append({'question': question, 'id': qid, 'is_impossible': False, 'answers': answers})
 paragraphs.append({'context': context, 'qas': qas})
 output['data'].append({'title': 'None', 'paragraphs': paragraphs})
 with open('data/test.json', 'w')as outfile:
json.dump(output, outfile )<load_pretrained>
binner = KBinsDiscretizer(n_bins=5,encode='ordinal', strategy='kmeans')
 train['age_bin_kmeans'] = pd.DataFrame(binner.fit_transform(train[['Age']].copy()))
test['age_bin_kmeans'] = pd.DataFrame(binner.fit_transform(test[['Age']].copy()))
!python /kaggle/input/pytorchtransformers/transformers-2.5.1/examples/run_squad.py \
--model_type roberta \
--model_name_or_path roberta-large \
--do_lower_case \
--do_train \
--do_eval \
--data_dir./data \
--cache_dir /kaggle/input/cached-roberta-large-pretrained/cache \
--train_file train.json \
--predict_file test.json \
--learning_rate 3e-5 \
--num_train_epochs 3 \
--max_seq_length 192 \
--doc_stride 64 \
--output_dir results_roberta_large \
--per_gpu_eval_batch_size=16 \
--per_gpu_train_batch_size=16 \
--save_steps=100000<string_transform>
binner = KBinsDiscretizer(n_bins=5,encode='ordinal', strategy='quantile')
 train['age_bin_quantile'] = pd.DataFrame(binner.fit_transform(train[['Age']].copy()))
test['age_bin_quantile'] = pd.DataFrame(binner.fit_transform(test[['Age']].copy()))
def post_process(selected):
return " ".join(set(selected.lower().split()))<load_from_csv>
useColumns = ['Type', 'Breed1', 'Breed2', 'Gender', 'Color1', 'Color2',
'Color3', 'MaturitySize', 'FurLength', 'Vaccinated', 'Dewormed',
'Sterilized', 'Health', 'Quantity', 'Fee', 'State',
'VideoAmt', 'PhotoAmt', 'AdoptionSpeed', 'has_name', 'has_topname', 'age_bin', 'age_bin_kmeans', 'age_bin_quantile']
predictions = json.load(open('results_roberta_large/predictions_.json', 'r'))
submission = pd.read_csv(open('/kaggle/input/tweet-sentiment-extraction/sample_submission.csv', 'r'))
for i in range(len(submission)) :
id_ = submission['textID'][i]
if pd_test['sentiment'][i] == 'neutral':
submission.loc[i, 'selected_text'] = pd_test['text'][i]
else:
submission.loc[i, 'selected_text'] = post_process(predictions[id_] )<save_to_csv>
cat_human_age = {1:0.5, 2:3, 3:4, 4:6, 5:8, 6:10, 7:12, 8:14, 9:15, 10:16, 11:17, 12:18, 24:24, 48:35, 72:42, 96:50, 120:60, 144:70, 168:80, 192:84 }
small_dog_human_age = {1:1, 2:2, 3:2.5, 4:3.5, 5:4.3, 6:5, 7:6.3, 8:7, 9:9, 10:11, 11:13, 12:15, 24:23, 48:32, 72:40, 96:48, 120:56, 144:64, 168:72, 192:80 }
normal_dog_human_age = {1:1, 2:2, 3:2.5, 4:3.5, 5:4.3, 6:5, 7:6.3, 8:7, 9:9, 10:11, 11:13, 12:15, 24:24, 48:34, 72:42, 96:51, 120:60, 144:69, 168:78, 192:87 }
big_dog_human_age = {1:1, 2:2, 3:2.5, 4:3.5, 5:4.3, 6:5, 7:6.3, 8:7, 9:9, 10:11, 11:13, 12:14, 24:22, 48:34, 72:45, 96:55, 120:66, 144:77, 168:88, 192:99 }
def human_age(row):
months = row['Age']
if months == 0:
return 0
if row['Type'] == 2:
if cat_human_age.get(months)is not None:
return cat_human_age.get(months)
 else:
if months < 25:
return 25
else:
return(25 +(( months/12)- 2)* 4)
 elif row['Type'] == 1 and row['MaturitySize'] == 1:
if small_dog_human_age.get(months)is not None:
return small_dog_human_age.get(months)
 else:
if months < 24:
return(months/12)* 11
else:
return(22 +(( months/12)- 2)* 4)
 elif row['Type'] == 1 and row['MaturitySize'] == 3:
if big_dog_human_age.get(months)is not None:
return big_dog_human_age.get(months)
 else:
if months < 24:
return(months/12)* 11
else:
return(22 +(( months/12)- 2)* 4)
 if normal_dog_human_age.get(months)is not None:
return normal_dog_human_age.get(months)
 else:
if months < 24:
return(months/12)* 11
else:
return(22 +(( months/12)- 2)* 4)
 def lifestage(row):
age = row['human_age']
if age < 10:
return 'Kitten/Puppy'
elif age < 25:
return 'Junior'
elif age < 40:
return 'Prime'
elif age < 60:
return 'Mature'
elif age < 74:
return 'Senior'
return 'Geriatic'
submission.to_csv('submission.csv', index=False )<set_options>
train['human_age'] = train.apply(human_age, axis=1)
 train['lifestage'] = train.apply(lifestage, axis=1)
 test['human_age'] = test.apply(human_age, axis=1)
 test['lifestage'] = test.apply(lifestage, axis=1 )
py.init_notebook_mode(connected=True)
 nltk.download('stopwords')
 stop=set(stopwords.words('english'))<load_from_csv>
top_3 = [
ensemble.AdaBoostClassifier() ,
ensemble.GradientBoostingClassifier() ,
XGBClassifier()
]
useColumns = ['Type', 'Breed1', 'Breed2', 'Gender', 'Color1', 'Color2',
'Color3', 'MaturitySize', 'FurLength', 'Vaccinated', 'Dewormed',
'Sterilized', 'Health', 'Quantity', 'Fee', 'State',
'VideoAmt', 'PhotoAmt', 'AdoptionSpeed', 'has_name', 'has_topname', 'age_bin',
'age_bin_kmeans', 'age_bin_quantile','human_age']
def read_train() :
train=pd.read_csv(".. /input/tweet-sentiment-extraction/train.csv")
 train['text']=train['text'].astype(str)
 train['selected_text']=train['selected_text'].astype(str)
 return train
def read_test() :
test=pd.read_csv(".. /input/tweet-sentiment-extraction/test.csv")
 test['text']=test['text'].astype(str)
 return test
def read_submission() :
test=pd.read_csv(".. /input/tweet-sentiment-extraction/sample_submission.csv")
 return test
train_df = read_train()
test_df = read_test()
submission_df = read_submission() 
<string_transform>
mapper = {'Kitten/Puppy':1, 'Junior':2, 'Prime':3, 'Mature':4,'Senior':5,'Geriatic':6}
train.lifestage.replace(mapper, inplace=True)
 test.lifestage.replace(mapper, inplace=True )
def remove_stopwords(text):
if text is not None:
tokens = [x for x in word_tokenize(text)if x not in stop]
return " ".join(tokens)
 else:
return None
def remove_URL(text):
url = re.compile(r'https?://\S+|www\.\S+')
 return url.sub(r'',text)
 def remove_html(text):
html=re.compile(r'<.*?>')
 return html.sub(r'',text)
 def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text)
 def remove_punct(text):
exclude = set(string.punctuation)
 s = ''.join(ch for ch in text if ch not in exclude)
 return s
def clean_df(df, train=True):
df["dirty_text"] = df['text']
df["text"] = df['text'].apply(lambda x : x.lower())
 df['text']=df['text'].apply(lambda x: remove_emoji(x))
df['text']=df['text'].apply(lambda x : remove_URL(x))
df['text']=df['text'].apply(lambda x : remove_html(x))
df['text'] =df['text'].apply(lambda x : remove_stopwords(x))
df['text']=df['text'].apply(lambda x : remove_punct(x))
df.text = df.text.replace('\s+', ' ', regex=True)
 if train:
df["selected_text"] = df['selected_text'].apply(lambda x : x.lower())
 df['selected_text']=df['selected_text'].apply(lambda x: remove_emoji(x))
df['selected_text']=df['selected_text'].apply(lambda x : remove_URL(x))
df['selected_text']=df['selected_text'].apply(lambda x : remove_html(x))
df['selected_text'] =df['selected_text'].apply(lambda x : remove_stopwords(x))
df['selected_text']=df['selected_text'].apply(lambda x : remove_punct(x))
df.selected_text = df.selected_text.replace('\s+', ' ', regex=True)
 return df 
<filter>
useColumns = ['Type', 'Breed1', 'Breed2', 'Gender', 'Color1', 'Color2',
'Color3', 'MaturitySize', 'FurLength', 'Vaccinated', 'Dewormed',
'Sterilized', 'Health', 'Quantity', 'Fee', 'State',
'VideoAmt', 'PhotoAmt', 'AdoptionSpeed', 'has_name', 'has_topname', 'age_bin',
'age_bin_kmeans', 'age_bin_quantile','human_age', 'lifestage']
def analyze_neutral_lenght(train_df):
neutral_df = train_df[train_df["sentiment"] == "neutral"]
equal_selected_with_text = neutral_df[neutral_df['selected_text'] == neutral_df["text"]]
print("Total number of neutral: {}".format(len(neutral_df)))
 print("Neutral with text equal to selected text: {}".format(len(equal_selected_with_text)))
 print("Ratio: {}".format(len(equal_selected_with_text)/len(neutral_df)))
 count = train_df.text.str.count(' ')<= 2
less_than_T = train_df.loc[count]
equal_selected_with_text = less_than_T[less_than_T['selected_text'] == less_than_T["text"]]
print("Total number of tweets with less than 3 words per text: {}".format(len(neutral_df)))
 print("Neutral with text equal to selected text: {}".format(len(equal_selected_with_text)))
 print("Ratio: {}".format(len(equal_selected_with_text)/len(less_than_T)))
 def post_process(submission_df, test_df):
index_to_selected_text = {}
for i, row in test_df.iterrows() :
_id = row[0]
text = row[1]
sentiment = row[2]
if len(text.split(" ")) <= 3 or sentiment == "neutral":
index_to_selected_text[_id] = text
submission_rows = submission_df.to_dict("records")
 new_rows = []
for row in submission_rows:
_id = row['textID']
if _id in index_to_selected_text:
new_row = deepcopy(row)
 new_row['selected_text'] = index_to_selected_text[_id]
else:
new_row = row
new_rows.append(new_row)
 return pd.DataFrame(new_rows)
 train_df = read_train()
test_df = read_test()
submission_df = read_submission()
analyze_neutral_lenght(train_df)
<feature_engineering>
def mixed_breed(row):
if row['Breed1'] == 307:
return 1
elif row['Breed2'] == 0:
return 0
elif row['Breed2'] != row['Breed1']:
return 1
else:
return 0
train['mixed_breed'] = train.apply(mixed_breed, axis=1)
 test['mixed_breed'] = test.apply(mixed_breed, axis=1 )
def analyze_cleaning(df, cleaning_lambda, field="selected_text"):
def diff_strings(row):
count = {}
A = row[0]
B = row[1]
if A is None:
A = "TEMP"
if B is None:
B = "TEMP"
for word in A.split() :
count[word] = count.get(word, 0)+ 1
for word in B.split() :
count[word] = count.get(word, 0)+ 1
diff = [word for word in count if count[word] == 1]
_max_len = max(len(A), len(B))
return len(diff)/_max_len
to_analyze = df[[field]]
to_analyze['cleaned_selected_text'] = to_analyze[field]
to_analyze["diff_ratio"] = None
to_analyze['cleaned_selected_text']=to_analyze['cleaned_selected_text'].apply(lambda x: cleaning_lambda(x))
to_analyze["diff_ratio"] = to_analyze.apply(lambda x: diff_strings(x), axis=1)
 print("Cleaning function used: {}".format(cleaning_lambda.__name__))
print("Average difference ratio: {}".format(to_analyze["diff_ratio"].mean()))
print("")
 train_df = read_train()
test_df = read_test()
cleaning_functions = [remove_stopwords, remove_URL, remove_html, remove_emoji, remove_punct]
for func in cleaning_functions:
analyze_cleaning(train_df, func, field="selected_text" )<string_transform>
top_1 = [
ensemble.GradientBoostingClassifier()
]
useColumns = ['Type', 'Breed1', 'Breed2', 'Gender', 'Color1', 'Color2',
'Color3', 'MaturitySize', 'FurLength', 'Vaccinated', 'Dewormed',
'Sterilized', 'Health', 'Quantity', 'Fee', 'State',
'VideoAmt', 'PhotoAmt', 'AdoptionSpeed', 'has_name', 'has_topname', 'age_bin',
'age_bin_kmeans', 'age_bin_quantile','human_age', 'lifestage','mixed_breed']
<save_to_csv>
train.groupby('Breed1')["AdoptionSpeed"].value_counts().head(5 )
<define_variables>
subtrain = train[train.Breed1 == 0]
cumulative_sum = subtrain.groupby('Breed1')["AdoptionSpeed"].cumsum() - subtrain["AdoptionSpeed"]
cumulative_count = subtrain.groupby('Breed1' ).cumcount()
print(cumulative_sum)
 print(cumulative_count)
 cumulative_sum/cumulative_count
tqdm.pandas()
BERT_PATH = '/kaggle/input/bert-base-uncased-huggingface-transformer/'
def find_offset(x,y):

 x_str = ' '.join(x)
 y_str = ' '.join(y)
 idx0=0
for ind in(i for i, e in enumerate(x_str)if e == y_str[0]):
if(x_str[ind: ind+len(y_str)] == y_str)or(x_str[ind: ind+len(y_str.replace('
idx0 = ind
idx1 = ind + len(y_str)- 1
break
t = 0
for offset,i in enumerate(x):
if t +len(i)+1>idx0:
break
t = t+len(i)+1
return offset
def create_targets(df, tokenizer):
df['t_text'] = df['text'].apply(lambda x: tokenizer.tokenize(str(x)))
 df['t_selected_text'] = df['selected_text'].apply(lambda x: tokenizer.tokenize(str(x)))
 def func(row):
x,y = row['t_text'],row['t_selected_text'][:]
_offset = 0
for offset in range(len(x)) :
_offset = offset
d = dict(zip(x[offset:],y))
check = [k==v for k,v in d.items() ]
if all(check)== True:
break
targets = [0]*_offset + [1]*len(y)+ [0]*(len(x)-_offset-len(y))
if len(targets)!= len(x):
offset = find_offset(x,y)
 targets = [0]*offset + [1]*len(y)+ [0] *(len(x)-offset-len(y))
return targets
df['targets'] = df.apply(func,axis=1)
 return df
def _convert_to_transformer_inputs(text, tokenizer, max_sequence_length):
def return_id(str1, str2, truncation_strategy, length):
inputs = tokenizer.encode_plus(str1, str2,
add_special_tokens=True,
max_length=length,
truncation_strategy=truncation_strategy)
 input_ids = inputs["input_ids"]
input_masks = [1] * len(input_ids)
 input_segments = inputs["token_type_ids"]
padding_length = length - len(input_ids)
 padding_id = tokenizer.pad_token_id
input_ids = input_ids +([padding_id] * padding_length)
 input_masks = input_masks +([0] * padding_length)
 input_segments = input_segments +([0] * padding_length)
 return [input_ids, input_masks, input_segments]
input_ids, input_masks, input_segments = return_id(text, None, 'longest_first', max_sequence_length)
 return [input_ids, input_masks, input_segments]
def compute_input_arrays(df, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df.iterrows()):
t = str(instance.text)
 ids, masks, segments= _convert_to_transformer_inputs(t,tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.asarray(input_ids, dtype=np.int32),
np.asarray(input_masks, dtype=np.int32),
np.asarray(input_segments, dtype=np.int32)]
def compute_output_arrays(df, columns):
return np.asarray(df[columns].values.tolist())
 def create_model() :
id = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 mask = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 attn = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 config = BertConfig()
config.output_hidden_states = True
bert_model = TFBertModel.from_pretrained(
 BERT_PATH+'bert-base-uncased-tf_model.h5', config=config)
 _,_, hidden_states = bert_model(id, attention_mask=mask, token_type_ids=attn)
 h12 = tf.reshape(hidden_states[-1][:,0],(-1,1,768))
h11 = tf.reshape(hidden_states[-2][:,0],(-1,1,768))
h10 = tf.reshape(hidden_states[-3][:,0],(-1,1,768))
h09 = tf.reshape(hidden_states[-4][:,0],(-1,1,768))
concat_hidden = tf.keras.layers.Concatenate(axis=2 )([h12, h11, h10, h09])
 x = tf.keras.layers.GlobalAveragePooling1D()(concat_hidden)
 x = tf.keras.layers.Dropout(0.2 )(x)
 x = tf.keras.layers.Dense(MAX_TARGET_LEN, activation='sigmoid' )(x)
 model = tf.keras.models.Model(inputs=[id, mask, attn], outputs=x)
 return model
def convert_pred_to_text(df,pred):
temp_output = []
for idx,p in enumerate(pred):
indexes = np.where(p>0.5)
 current_text = df['t_text'][idx]
if len(indexes[0])>0:
start = indexes[0][0]
end = indexes[0][-1]
else:
start = 0
end = len(current_text)
 if end >= len(current_text):
end = len(current_text)
 temp_output.append(' '.join(current_text[start:end+1]))
return temp_output
def correct_op(row):
placeholder = row['temp_output']
for original_token in str(row['text'] ).split() :
token_str = ' '.join(tokenizer.tokenize(original_token))
placeholder = placeholder.replace(token_str,original_token,1)
 return placeholder
def replacer(row):
if row['sentiment'] == 'neutral':
return row['text']
else:
return row['temp_output2'] 
<install_modules>
def cat_mean_encoding(df, column):
cumulative_sum = df.groupby(column)["AdoptionSpeed"].cumsum() - df["AdoptionSpeed"]
cumulative_count = df.groupby(column ).cumcount()
df[column + "_mean_encoding"] = cumulative_sum/cumulative_count
df[column + "_mean_encoding"].fillna(df.AdoptionSpeed.mean() , inplace=True )
!mkdir -p data
!pip install '/kaggle/input/simple-transformers-pypi/seqeval-0.0.12-py3-none-any.whl' -q
!pip install '/kaggle/input/simple-transformers-pypi/simpletransformers-0.22.1-py3-none-any.whl' -q
use_cuda = True
def find_all(input_str, search_str):
l1 = []
length = len(input_str)
 index = 0
while index < length:
i = input_str.find(search_str, index)
 if i == -1:
return l1
l1.append(i)
 index = i + 1
return l1
def do_qa_train(train):
output = {}
output['version'] = 'v1.0'
output['data'] = []
paragraphs = []
for line in train:
context = line[1]
qas = []
question = line[-1]
qid = line[0]
answers = []
answer = line[2]
if type(answer)!= str or type(context)!= str or type(question)!= str:
print(context, type(context))
print(answer, type(answer))
print(question, type(question))
continue
answer_starts = find_all(context, answer)
 for answer_start in answer_starts:
answers.append({'answer_start': answer_start, 'text': answer.lower() })
 break
qas.append({'question': question, 'id': qid, 'is_impossible': False, 'answers': answers})
 paragraphs.append({'context': context.lower() , 'qas': qas})
 output['data'].append({'title': 'None', 'paragraphs': paragraphs})
 return paragraphs
def do_qa_test(test):
paragraphs = []
for line in test:
context = line[1]
qas = []
question = line[-1]
qid = line[0]
if type(context)!= str or type(question)!= str:
print(context, type(context))
print(answer, type(answer))
print(question, type(question))
continue
answers = []
answers.append({'answer_start': 1000000, 'text': '__None__'})
 qas.append({'question': question, 'id': qid, 'is_impossible': False, 'answers': answers})
 paragraphs.append({'context': context.lower() , 'qas': qas})
 output['data'].append({'title': 'None', 'paragraphs': paragraphs})
 return paragraphs
train_df = read_train()
test_df = read_test()
submission_df_distil = read_submission()
train = np.array(train_df)
 test = np.array(test_df)
 qa_train = do_qa_train(train)
 with open('data/train.json', 'w')as outfile:
json.dump(qa_train, outfile)
 output = {}
output['version'] = 'v1.0'
output['data'] = []
qa_test = do_qa_test(test)
 with open('data/test.json', 'w')as outfile:
json.dump(qa_test, outfile)
 MODEL_PATH = '/kaggle/input/transformers-pretrained-distilbert/distilbert-base-uncased-distilled-squad/'
model = QuestionAnsweringModel('distilbert',
MODEL_PATH,
args={"reprocess_input_data": True,
"overwrite_output_dir": True,
"learning_rate": 8e-05,
"num_train_epochs": 3,
"max_seq_length": 192,
"weight_decay": 0.001,
"doc_stride": 64,
"save_model_every_epoch": False,
"fp16": False,
"do_lower_case": True,
'max_query_length': 8,
'max_answer_length': 150
},
use_cuda=use_cuda)
 model.train_model('data/train.json')
 predictions = model.predict(qa_test)
 predictions_df = pd.DataFrame.from_dict(predictions)
 submission_df_distil['selected_text'] = predictions_df['answer']
submission_df_distil = post_process(submission_df_distil, test_df)
<install_modules>
class KFoldTargetEncoderTrain(base.BaseEstimator, base.TransformerMixin):
def __init__(self, colnames,targetName,n_fold=5,verbosity=True,discardOriginal_col=False):
self.colnames = colnames
self.targetName = targetName
self.n_fold = n_fold
self.verbosity = verbosity
self.discardOriginal_col = discardOriginal_col
def fit(self, X, y=None):
return self
def transform(self,X):
assert(type(self.targetName)== str)
 assert(type(self.colnames)== str)
 assert(self.colnames in X.columns)
 assert(self.targetName in X.columns)
 mean_of_target = X[self.targetName].mean()
kf = StratifiedKFold(n_splits=self.n_fold, shuffle=True)
 col_mean_name = self.colnames + '_' + 'Kfold_Target_Enc'
X[col_mean_name] = np.nan
for tr_ind, val_ind in kf.split(X,X[self.targetName]):
X_tr, X_val = X.iloc[tr_ind], X.iloc[val_ind]
X.loc[X.index[val_ind], col_mean_name] = X_val[self.colnames].map(X_tr.groupby(self.colnames)[self.targetName].mean())
 X[col_mean_name].fillna(mean_of_target, inplace = True)
 if self.verbosity:
encoded_feature = X[col_mean_name].values
print('Correlation between the new feature, {} and, {} is {}.'.format(col_mean_name,
self.targetName,
np.corrcoef(X[self.targetName].values, encoded_feature)[0][1]))
if self.discardOriginal_col:
X = X.drop(self.targetName, axis=1)
 return X
class KFoldTargetEncoderTest(base.BaseEstimator, base.TransformerMixin):
def __init__(self,train,colNames,encodedName):
self.train = train
self.colNames = colNames
self.encodedName = encodedName
def fit(self, X, y=None):
return self
def transform(self,X):
mean = self.train[[self.colNames,self.encodedName]].groupby(self.colNames ).mean().reset_index()
dd = {}
for index, row in mean.iterrows() :
dd[row[self.colNames]] = row[self.encodedName]
X[self.encodedName] = X[self.colNames]
X[self.encodedName] = X[self.encodedName].map(dd)
 return X
!mkdir -p data
!pip install '/kaggle/input/simple-transformers-pypi/seqeval-0.0.12-py3-none-any.whl' -q
!pip install '/kaggle/input/simple-transformers-pypi/simpletransformers-0.22.1-py3-none-any.whl' -q
use_cuda = True
def find_all(input_str, search_str):
l1 = []
length = len(input_str)
 index = 0
while index < length:
i = input_str.find(search_str, index)
 if i == -1:
return l1
l1.append(i)
 index = i + 1
return l1
def do_qa_train(train):
output = {}
output['version'] = 'v1.0'
output['data'] = []
paragraphs = []
for line in train:
context = line[1]
qas = []
question = line[-1]
qid = line[0]
answers = []
answer = line[2]
if type(answer)!= str or type(context)!= str or type(question)!= str:
print(context, type(context))
print(answer, type(answer))
print(question, type(question))
continue
answer_starts = find_all(context, answer)
 for answer_start in answer_starts:
answers.append({'answer_start': answer_start, 'text': answer.lower() })
 break
qas.append({'question': question, 'id': qid, 'is_impossible': False, 'answers': answers})
 paragraphs.append({'context': context.lower() , 'qas': qas})
 output['data'].append({'title': 'None', 'paragraphs': paragraphs})
 return paragraphs
def do_qa_test(test):
paragraphs = []
for line in test:
context = line[1]
qas = []
question = line[-1]
qid = line[0]
if type(context)!= str or type(question)!= str:
print(context, type(context))
print(answer, type(answer))
print(question, type(question))
continue
answers = []
answers.append({'answer_start': 1000000, 'text': '__None__'})
 qas.append({'question': question, 'id': qid, 'is_impossible': False, 'answers': answers})
 paragraphs.append({'context': context.lower() , 'qas': qas})
 output['data'].append({'title': 'None', 'paragraphs': paragraphs})
 return paragraphs
train_df = read_train()
test_df = read_test()
submission_df_albert = read_submission()
train = np.array(train_df)
 test = np.array(test_df)
 qa_train = do_qa_train(train)
 with open('data/train.json', 'w')as outfile:
json.dump(qa_train, outfile)
 output = {}
output['version'] = 'v1.0'
output['data'] = []
qa_test = do_qa_test(test)
 with open('data/test.json', 'w')as outfile:
json.dump(qa_test, outfile)
 MODEL_PATH = '/kaggle/input/pretrained-albert-pytorch/albert-base-v2/'
model = QuestionAnsweringModel('albert',
MODEL_PATH,
args={"reprocess_input_data": True,
"overwrite_output_dir": True,
"learning_rate": 8e-05,
"num_train_epochs": 3,
"max_seq_length": 192,
"weight_decay": 0.001,
"doc_stride": 64,
"save_model_every_epoch": False,
"fp16": False,
"do_lower_case": True,
'max_query_length': 8,
'max_answer_length': 150
},
use_cuda=use_cuda)
 model.train_model('data/train.json')
 predictions = model.predict(qa_test)
 predictions_df = pd.DataFrame.from_dict(predictions)
 submission_df_albert['selected_text'] = predictions_df['answer']
submission_df_albert = post_process(submission_df_albert, test_df)
<install_modules>
targetc = KFoldTargetEncoderTrain('Breed1','AdoptionSpeed',n_fold=5)
 train = targetc.fit_transform(train)
 targetc = KFoldTargetEncoderTrain('Breed2','AdoptionSpeed',n_fold=5)
 train = targetc.fit_transform(train)
 test_targetc = KFoldTargetEncoderTest(train,'Breed1','Breed1_Kfold_Target_Enc')
 test= test_targetc.fit_transform(test)
 test_targetc = KFoldTargetEncoderTest(train,'Breed2','Breed2_Kfold_Target_Enc')
 test= test_targetc.fit_transform(test)
!mkdir -p data
!pip install '/kaggle/input/simple-transformers-pypi/seqeval-0.0.12-py3-none-any.whl' -q
!pip install '/kaggle/input/simple-transformers-pypi/simpletransformers-0.22.1-py3-none-any.whl' -q
use_cuda = True
def find_all(input_str, search_str):
l1 = []
length = len(input_str)
 index = 0
while index < length:
i = input_str.find(search_str, index)
 if i == -1:
return l1
l1.append(i)
 index = i + 1
return l1
def do_qa_train(train):
output = {}
output['version'] = 'v1.0'
output['data'] = []
paragraphs = []
for line in train:
context = line[1]
qas = []
question = line[-1]
qid = line[0]
answers = []
answer = line[2]
if type(answer)!= str or type(context)!= str or type(question)!= str:
print(context, type(context))
print(answer, type(answer))
print(question, type(question))
continue
answer_starts = find_all(context, answer)
 for answer_start in answer_starts:
answers.append({'answer_start': answer_start, 'text': answer.lower() })
 break
qas.append({'question': question, 'id': qid, 'is_impossible': False, 'answers': answers})
 paragraphs.append({'context': context.lower() , 'qas': qas})
 output['data'].append({'title': 'None', 'paragraphs': paragraphs})
 return paragraphs
def do_qa_test(test):
paragraphs = []
for line in test:
context = line[1]
qas = []
question = line[-1]
qid = line[0]
if type(context)!= str or type(question)!= str:
print(context, type(context))
print(answer, type(answer))
print(question, type(question))
continue
answers = []
answers.append({'answer_start': 1000000, 'text': '__None__'})
 qas.append({'question': question, 'id': qid, 'is_impossible': False, 'answers': answers})
 paragraphs.append({'context': context.lower() , 'qas': qas})
 output['data'].append({'title': 'None', 'paragraphs': paragraphs})
 return paragraphs
train_df = read_train()
test_df = read_test()
submission_df_bert = read_submission()
train = np.array(train_df)
 test = np.array(test_df)
 qa_train = do_qa_train(train)
 with open('data/train.json', 'w')as outfile:
json.dump(qa_train, outfile)
 output = {}
output['version'] = 'v1.0'
output['data'] = []
qa_test = do_qa_test(test)
 with open('data/test.json', 'w')as outfile:
json.dump(qa_test, outfile)
 MODEL_PATH = '/kaggle/input/bert-base-uncased/'
model = QuestionAnsweringModel('bert',
MODEL_PATH,
args={"reprocess_input_data": True,
"overwrite_output_dir": True,
"learning_rate": 8e-05,
"num_train_epochs": 3,
"max_seq_length": 192,
"weight_decay": 0.001,
"doc_stride": 64,
"save_model_every_epoch": False,
"fp16": False,
"do_lower_case": True,
'max_query_length': 8,
'max_answer_length': 150
},
use_cuda=use_cuda)
 model.train_model('data/train.json')
 predictions = model.predict(qa_test)
 predictions_df = pd.DataFrame.from_dict(predictions)
 submission_df_bert['selected_text'] = predictions_df['answer']
submission_df_bert = post_process(submission_df_bert, test_df)
<merge>
cat_mean_encoding(train, 'Breed1')
 cat_mean_encoding(train, 'Breed2')
 test_targetc = KFoldTargetEncoderTest(train,'Breed1','Breed1_mean_encoding')
 test= test_targetc.fit_transform(test)
 test_targetc = KFoldTargetEncoderTest(train,'Breed2','Breed2_mean_encoding')
 test= test_targetc.fit_transform(test )
sentiment_analyzer = SentimentIntensityAnalyzer()
def merge_predictions(submission_distil, sumbission_albert, submission_bert):
def merge(row):
pred_1 = set(str(row[1] ).split())
 pred_2 = set(str(row[2] ).split())
 pred_3 = set(str(row[3] ).split())
 res = []
res = list(pred_1.intersection(pred_2, pred_3))
if len(res)== 0:
res = list(pred_1.union(pred_2, pred_3))
return " ".join(res)
 def merge_compound(row):
pred_1 = str(row[1])
 pred_2 = str(row[2])
 pred_3 = str(row[3])
 preds = [pred_1, pred_2, pred_3]
comp_1 = abs(sentiment_analyzer.polarity_scores(pred_1 ).get("compound",0))
comp_2 = abs(sentiment_analyzer.polarity_scores(pred_2 ).get("compound",0))
comp_3 = abs(sentiment_analyzer.polarity_scores(pred_3 ).get("compound",0))
comps = [comp_1, comp_2, comp_3]
best_comp = comps.index(max(comps))
return preds[best_comp]
def capitalize_selected(df):
rows = df.to_dict("records")
 new_rows = []
for r in rows:
text = r["text"]
lower_text = text.lower()
pred_distil = r["selected_text_distil"]
pred_albert = r["selected_text_albert"]
pred_bert = r["selected_text_bert"]
new_pred_distil = text[lower_text.find(pred_distil):len(pred_distil)]
new_pred_albert = text[lower_text.find(pred_albert):len(pred_albert)]
new_pred_bert = text[lower_text.find(pred_bert):len(pred_bert)]
new_row = {
"textID": r['textID'],
"selected_text_distil": new_pred_distil,
"selected_text_albert": new_pred_albert,
"selected_text_bert": new_pred_bert
}
new_rows.append(new_row)
 return pd.DataFrame(new_rows)
 sumbission_albert = sumbission_albert.rename({'selected_text': 'selected_text_albert'}, axis=1)
 del sumbission_albert["textID"]
submission_distil = submission_distil.rename({'selected_text': 'selected_text_distil'}, axis=1)
 submission_bert = submission_bert.rename({'selected_text': 'selected_text_bert'}, axis=1)
 del submission_bert["textID"]
final_sub = pd.concat([submission_distil, sumbission_albert, submission_bert], axis=1)
 final_sub["selected_text"] = final_sub.apply(lambda x: merge_compound(x), axis=1)
 del final_sub["selected_text_albert"]
del final_sub["selected_text_distil"]
del final_sub["selected_text_bert"]
return final_sub

 submission = merge_predictions(submission_df_distil, submission_df_albert, submission_df_bert)
 submission.to_csv('submission.csv', index=False)
<import_modules>
print(train.Breed1_Kfold_Target_Enc.isna().sum())
 print(train.Breed2_Kfold_Target_Enc.isna().sum())
 print(test.Breed1_Kfold_Target_Enc.isna().sum())
 print(test.Breed2_Kfold_Target_Enc.isna().sum())
 print(train.Breed1_mean_encoding.isna().sum())
 print(train.Breed2_mean_encoding.isna().sum())
 print(test.Breed1_mean_encoding.isna().sum())
 print(test.Breed2_mean_encoding.isna().sum())
 test[~test.Breed1.isin(train.Breed1)].head(5 )
import pandas as pd
import string
from nltk.corpus import stopwords
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import log_loss
from sklearn.metrics import accuracy_score
from scipy.sparse import hstack<load_from_csv>
test.Breed1_Kfold_Target_Enc.fillna(train.AdoptionSpeed.mean() , inplace=True)
 test.Breed2_Kfold_Target_Enc.fillna(train.AdoptionSpeed.mean() , inplace=True)
 test.Breed1_mean_encoding.fillna(train.AdoptionSpeed.mean() , inplace=True)
 test.Breed2_mean_encoding.fillna(train.AdoptionSpeed.mean() , inplace=True )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' ).fillna('unknown' )<prepare_x_and_y>
useColumns = ['Type', 'Breed1_Kfold_Target_Enc','Breed2_Kfold_Target_Enc','Gender', 'Color1', 'Color2',
'Color3', 'MaturitySize', 'FurLength', 'Vaccinated', 'Dewormed',
'Sterilized', 'Health', 'Quantity', 'Fee', 'State',
'VideoAmt', 'PhotoAmt', 'AdoptionSpeed', 'has_name', 'has_topname', 'age_bin',
'age_bin_kmeans', 'age_bin_quantile','human_age', 'lifestage','mixed_breed',
'Breed1_mean_encoding', 'Breed2_mean_encoding']
class_names = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']
X_train = train['comment_text']
y = train[class_names]
X_test = test['comment_text']<concatenate>
rescuer_count = train.groupby(['RescuerID'])['PetID'].count().reset_index()
rescuer_count_dict = rescuer_count.set_index('RescuerID' ).T.to_dict('list')
 train.RescuerID = train.RescuerID.apply(lambda x: np.log(rescuer_count_dict.get(x)[0]))
binner = KBinsDiscretizer(n_bins=6,encode='ordinal', strategy='kmeans')
 train['rescuer_bin_kmeans'] = pd.DataFrame(binner.fit_transform(train[['RescuerID']].copy()))
targetc = KFoldTargetEncoderTrain('rescuer_bin_kmeans','AdoptionSpeed',n_fold=5)
 train = targetc.fit_transform(train)
 rescuer_count = test.groupby(['RescuerID'])['PetID'].count().reset_index()
rescuer_count_dict = rescuer_count.set_index('RescuerID' ).T.to_dict('list')
 test.RescuerID = test.RescuerID.apply(lambda x: np.log(rescuer_count_dict.get(x)[0]))
binner = KBinsDiscretizer(n_bins=5,encode='ordinal', strategy='kmeans')
 test['rescuer_bin_kmeans'] = pd.DataFrame(binner.fit_transform(test[['RescuerID']].copy()))
test_targetc = KFoldTargetEncoderTest(train,'rescuer_bin_kmeans','rescuer_bin_kmeans_Kfold_Target_Enc')
 test= test_targetc.fit_transform(test )
all_text = pd.concat([X_train, X_test] )<choose_model_class>
useColumns = ['Type', 'Breed1_Kfold_Target_Enc','Breed2_Kfold_Target_Enc','Gender', 'Color1', 'Color2',
'Color3', 'MaturitySize', 'FurLength', 'Vaccinated', 'Dewormed',
'Sterilized', 'Health', 'Quantity', 'Fee', 'State',
'VideoAmt', 'PhotoAmt', 'AdoptionSpeed', 'has_name', 'has_topname', 'age_bin',
'age_bin_kmeans', 'age_bin_quantile','human_age', 'lifestage','mixed_breed',
'Breed1_mean_encoding', 'Breed2_mean_encoding','rescuer_bin_kmeans','rescuer_bin_kmeans_Kfold_Target_Enc']
vectorizer = TfidfVectorizer(
 min_df=3, max_df=0.8,
ngram_range=(1, 2),
strip_accents='unicode',
sublinear_tf=True, )<train_model>
targetc = KFoldTargetEncoderTrain('State','AdoptionSpeed',n_fold=5)
 train = targetc.fit_transform(train)
 test_targetc = KFoldTargetEncoderTest(train,'State','State_Kfold_Target_Enc')
 test= test_targetc.fit_transform(test )
vectorizer.fit(all_text )<feature_engineering>
useColumns = ['Type', 'Breed1_Kfold_Target_Enc','Breed2_Kfold_Target_Enc','Gender', 'Color1', 'Color2',
'Color3', 'MaturitySize', 'FurLength', 'Vaccinated', 'Dewormed',
'Sterilized', 'Health', 'Quantity', 'Fee','State_Kfold_Target_Enc','state_gdp','state_population','state_area',
'VideoAmt', 'PhotoAmt', 'AdoptionSpeed', 'has_name', 'has_topname', 'age_bin',
'age_bin_kmeans', 'age_bin_quantile','human_age', 'lifestage','mixed_breed',
'Breed1_mean_encoding', 'Breed2_mean_encoding','rescuer_bin_kmeans','rescuer_bin_kmeans_Kfold_Target_Enc'
]
train_word = vectorizer.transform(X_train)
 test_word = vectorizer.transform(X_test )<choose_model_class>
targetc = KFoldTargetEncoderTrain('Color1','AdoptionSpeed',n_fold=5)
 train = targetc.fit_transform(train)
 test_targetc = KFoldTargetEncoderTest(train,'Color1','Color1_Kfold_Target_Enc')
 test= test_targetc.fit_transform(test)
 targetc = KFoldTargetEncoderTrain('Color2','AdoptionSpeed',n_fold=5)
 train = targetc.fit_transform(train)
 test_targetc = KFoldTargetEncoderTest(train,'Color2','Color2_Kfold_Target_Enc')
 test= test_targetc.fit_transform(test)
 targetc = KFoldTargetEncoderTrain('Color3','AdoptionSpeed',n_fold=5)
 train = targetc.fit_transform(train)
 test_targetc = KFoldTargetEncoderTest(train,'Color3','Color3_Kfold_Target_Enc')
 test= test_targetc.fit_transform(test )
char_vectorizer = TfidfVectorizer(
 sublinear_tf=True,
strip_accents='unicode',
analyzer='char',
ngram_range=(1, 4),
max_features=20000 )<train_model>
useColumns = ['Type', 'Breed1_Kfold_Target_Enc','Breed2_Kfold_Target_Enc','Gender',
'Color1_Kfold_Target_Enc', 'Color2_Kfold_Target_Enc',
'Color3_Kfold_Target_Enc', 'MaturitySize', 'FurLength', 'Vaccinated', 'Dewormed',
'Sterilized', 'Health', 'Quantity', 'Fee','State_Kfold_Target_Enc','state_gdp','state_population','state_area',
'VideoAmt', 'PhotoAmt', 'AdoptionSpeed', 'has_name', 'has_topname', 'age_bin',
'age_bin_kmeans', 'age_bin_quantile','human_age', 'lifestage','mixed_breed',
'Breed1_mean_encoding', 'Breed2_mean_encoding','rescuer_bin_kmeans','rescuer_bin_kmeans_Kfold_Target_Enc'
]
char_vectorizer.fit(all_text )<feature_engineering>
train.Description.fillna('none', inplace=True)
 test.Description.fillna('none', inplace=True)
 train['desc_length'] = train.Description.apply(len)
 test['desc_length'] = test.Description.apply(len )
train_char = char_vectorizer.transform(X_train)
 test_char = char_vectorizer.transform(X_test )<concatenate>
tfv = TfidfVectorizer(min_df=2, max_features=None,
strip_accents='unicode', analyzer='word', token_pattern=r'(?u)\b\w+\b',
ngram_range=(1, 4), use_idf=1, smooth_idf=1, sublinear_tf=1)
 svd_ = TruncatedSVD(n_components=5)
 train_test = pd.concat([train, test], ignore_index=True, sort=False)
 tfidf_col = tfv.fit_transform(train_test.Description)
 svd_col = svd_.fit_transform(tfidf_col)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('TFIDF_')
 pcas = pd.concat([train_test, svd_col], axis=1)
 train = pcas.loc[np.isfinite(pcas.AdoptionSpeed), :]
test = pcas.loc[~np.isfinite(pcas.AdoptionSpeed), :]
train_feat = hstack([train_char, train_word])
 test_feat = hstack([test_char, test_word] )<load_from_csv>
useColumns = ['Type', 'Breed1_Kfold_Target_Enc','Breed2_Kfold_Target_Enc','Gender',
'Color1_Kfold_Target_Enc', 'Color2_Kfold_Target_Enc',
'Color3_Kfold_Target_Enc', 'MaturitySize', 'FurLength', 'Vaccinated', 'Dewormed',
'Sterilized', 'Health', 'Quantity', 'Fee','State_Kfold_Target_Enc','state_gdp','state_population','state_area',
'VideoAmt', 'PhotoAmt', 'AdoptionSpeed', 'has_name', 'has_topname', 'age_bin',
'age_bin_kmeans', 'age_bin_quantile','human_age', 'lifestage','mixed_breed',
'Breed1_mean_encoding', 'Breed2_mean_encoding','rescuer_bin_kmeans','rescuer_bin_kmeans_Kfold_Target_Enc',
'desc_length', 'TFIDF_0','TFIDF_1','TFIDF_2','TFIDF_3','TFIDF_4' ]
y_pred = pd.read_csv('.. /input/sample_submission.csv')
 for c in class_names:
clf = LogisticRegression(C=4)
 clf.fit(train_feat, y[c])
 y_pred[c] = clf.predict_proba(test_feat)[:,1]
pred_train = clf.predict_proba(train_feat)[:,1]
print('log loss ',c, ':', log_loss(y[c], pred_train))<save_to_csv>
useColumns = ['Type', 'Breed1_Kfold_Target_Enc','Breed2_Kfold_Target_Enc','Gender',
'Color1_Kfold_Target_Enc', 'Color2_Kfold_Target_Enc',
'Color3_Kfold_Target_Enc', 'MaturitySize', 'FurLength', 'Vaccinated', 'Dewormed',
'Sterilized', 'Health', 'Quantity', 'Fee','State_Kfold_Target_Enc','state_gdp','state_population','state_area',
'VideoAmt', 'PhotoAmt', 'has_name', 'has_topname', 'age_bin',
'age_bin_kmeans', 'age_bin_quantile','human_age', 'lifestage','mixed_breed',
'Breed1_mean_encoding', 'Breed2_mean_encoding','rescuer_bin_kmeans','rescuer_bin_kmeans_Kfold_Target_Enc'
]
features = train[useColumns]
kappa_score = make_scorer(cohen_kappa_score, weights='quadratic')
 for alg in []:
kf = StratifiedKFold(n_splits=5, random_state=42, shuffle=True)
 rfecv = RFECV(estimator=alg, step=1, cv=kf, scoring=kappa_score, n_jobs=-1)
 rfecv.fit(features,train['AdoptionSpeed'])
 print("Optimal number of features : %d" % rfecv.n_features_)
 print(rfecv.support_)
 print(features.columns[rfecv.support_])
 plt.figure()
plt.xlabel("Number of features selected")
 plt.ylabel("Cross validation score(nb of correct classifications)")
 plt.plot(range(1, len(rfecv.grid_scores_)+ 1), rfecv.grid_scores_)
 plt.show()
y_pred.to_csv("mysubmission.csv", index=False )<train_model>
useColumns = ['Breed1_Kfold_Target_Enc', 'Breed2_Kfold_Target_Enc', 'FurLength',
'Sterilized', 'State_Kfold_Target_Enc', 'PhotoAmt', 'age_bin',
'human_age', 'Breed1_mean_encoding', 'Breed2_mean_encoding',
'rescuer_bin_kmeans', 'rescuer_bin_kmeans_Kfold_Target_Enc']
params = {
'n_estimators': [50, 100, 200],
'learning_rate' : [0.01,0.05,0.1,0.3,1],
'base_estimator': [ensemble.AdaBoostClassifier() , ensemble.GradientBoostingClassifier() ]
}
X_train = train[useColumns]
y_train = train['AdoptionSpeed']
kf = StratifiedKFold(n_splits=5, random_state=42, shuffle=True)
 cv_split = kf.split(X_train, y_train)
 kappa_score = make_scorer(cohen_kappa_score, weights='quadratic')
 tuning = GridSearchCV(estimator = ensemble.AdaBoostClassifier() , param_grid = params,
scoring=kappa_score,cv=5, n_jobs=-1,iid=False)
<import_modules>
params = {'learning_rate':[0.15,0.1,0.05,0.01,0.005,0.001],
'n_estimators':[100,250,500,750,1000,1250,1500,1750],
'max_depth':[2,3,4,5,6,7],
'min_samples_split':[2,4,6,8,10,20,40,60,100],
'min_samples_leaf':[1,3,5,7,9],
'max_features':[2,3,4,5,6,7],
'subsample':[0.7,0.75,0.8,0.85,0.9,0.95,1]}
kf = StratifiedKFold(n_splits=5, random_state=42, shuffle=True)
 kappa_score = make_scorer(cohen_kappa_score, weights='quadratic')
 tuning = GridSearchCV(estimator = ensemble.GradientBoostingClassifier() , param_grid = params,
scoring=kappa_score,cv=kf, n_jobs=-1,iid=False)
 train_cleaned = train.drop(['Name','human_age','lifestage','RescuerID','Description','PetID', 'dataset_type'],axis=1)
 y_train = train_cleaned['AdoptionSpeed']
X_train = train_cleaned.drop(['AdoptionSpeed'],axis=1)
import pandas as pd<import_modules>
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):

 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)]
for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):

 if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):

 rater_a = y
rater_b = y_pred
min_rating=None
max_rating=None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j]
/ num_scored_items)
 d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator)
 class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 return -cohen_kappa_score(y, preds, weights='quadratic')
 def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X = X, y = y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 return preds
def coefficients(self):
return self.coef_['x']
import pandas as pd<load_from_csv>
useColumns = ['Type', 'Breed1_Kfold_Target_Enc','Breed2_Kfold_Target_Enc','Gender',
'Color1_Kfold_Target_Enc', 'Color2_Kfold_Target_Enc',
'Color3_Kfold_Target_Enc', 'MaturitySize', 'FurLength', 'Vaccinated', 'Dewormed',
'Sterilized', 'Health', 'Quantity', 'Fee','State_Kfold_Target_Enc','state_gdp','state_population','state_area',
'VideoAmt', 'PhotoAmt','has_name', 'has_topname', 'age_bin',
'age_bin_kmeans', 'age_bin_quantile','human_age', 'lifestage','mixed_breed',
'Breed1_mean_encoding', 'Breed2_mean_encoding','rescuer_bin_kmeans','rescuer_bin_kmeans_Kfold_Target_Enc',
'desc_length', 'TFIDF_0','TFIDF_1','TFIDF_2','TFIDF_3','TFIDF_4' ]
target = train.AdoptionSpeed.astype('int')
 X_train=train[useColumns]
X_test=test[useColumns]
glove = pd.read_csv(".. /input/nb-svm-strong-linear-baseline/submission.csv")
 subb = pd.read_csv('.. /input/fasttext-like-baseline-with-keras-lb-0-053/submission_bn_fasttext.csv')
 ave = pd.read_csv('.. /input/toxic-avenger/submission.csv')
 lstm = pd.read_csv('.. /input/toxicfiles/baselinelstm0069.csv')
 svm = pd.read_csv(".. /input/toxicfiles/lstmglove0072ge.csv" )<drop_column>
useColumns = ['AdoptionSpeed','Type', 'Breed1_Kfold_Target_Enc','Breed2_Kfold_Target_Enc','Gender',
'Color1_Kfold_Target_Enc', 'Color2_Kfold_Target_Enc',
'Color3_Kfold_Target_Enc', 'MaturitySize', 'FurLength', 'Vaccinated', 'Dewormed',
'Sterilized', 'Health', 'Quantity', 'Fee','State_Kfold_Target_Enc','state_gdp','state_population','state_area',
'VideoAmt', 'PhotoAmt','has_name', 'has_topname', 'age_bin',
'age_bin_kmeans', 'age_bin_quantile','human_age', 'lifestage','mixed_breed',
'Breed1_mean_encoding', 'Breed2_mean_encoding','rescuer_bin_kmeans','rescuer_bin_kmeans_Kfold_Target_Enc',
'desc_length', 'TFIDF_0','TFIDF_1','TFIDF_2','TFIDF_3','TFIDF_4' ]
target = train.AdoptionSpeed.astype('int')
 X_train=train[useColumns]
X_test=test[useColumns].drop(['AdoptionSpeed'], axis=1 )
col = col.tolist()
col.remove('id' )<feature_engineering>
xgb_params = {
'eval_metric': 'rmse',
'seed': 1337,
'silent': 1,
}
for i in col:
ble[i] =(subb[i] + lstm[i] + glove[i] + svm[i] + ave[i])/ 5<save_to_csv>
def run_xgb(params, X_train, X_test):
n_splits = 5
verbose_eval = 1000
num_rounds = 30000
early_stop = 500
kf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=1337)
 oof_train = np.zeros(( X_train.shape[0]))
oof_test = np.zeros(( X_test.shape[0], n_splits))
i = 0
for train_idx, valid_idx in kf.split(X_train, X_train['AdoptionSpeed'].values):
X_tr = X_train.iloc[train_idx, :]
X_val = X_train.iloc[valid_idx, :]
y_tr = X_tr['AdoptionSpeed'].values
X_tr = X_tr.drop(['AdoptionSpeed'], axis=1)
 y_val = X_val['AdoptionSpeed'].values
X_val = X_val.drop(['AdoptionSpeed'], axis=1)
 d_train = xgb.DMatrix(data=X_tr, label=y_tr, feature_names=X_tr.columns)
 d_valid = xgb.DMatrix(data=X_val, label=y_val, feature_names=X_val.columns)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
model = xgb.train(dtrain=d_train, num_boost_round=num_rounds, evals=watchlist,
early_stopping_rounds=early_stop, verbose_eval=verbose_eval, params=params)
 valid_pred = model.predict(xgb.DMatrix(X_val, feature_names=X_val.columns), ntree_limit=model.best_ntree_limit)
 test_pred = model.predict(xgb.DMatrix(X_test, feature_names=X_test.columns), ntree_limit=model.best_ntree_limit)
 oof_train[valid_idx] = valid_pred
oof_test[:, i] = test_pred
i += 1
return model, oof_train, oof_test
ble.to_csv('blend_sub.csv', index = False )<set_options>
model, oof_train, oof_test = run_xgb(xgb_params, X_train, X_test )
print('hi' )<define_variables>
optR = OptimizedRounder()
optR.fit(oof_train, X_train['AdoptionSpeed'].values)
 coefficients = optR.coefficients()
valid_pred = optR.predict(oof_train, coefficients)
 qwk = quadratic_weighted_kappa(X_train['AdoptionSpeed'].values, valid_pred)
 print("QWK = ", qwk )
path = '.. /input/'
comp = 'jigsaw-toxic-comment-classification-challenge/'
EMBEDDING_FILE=f'{path}glove6b50d/glove.6B.50d.txt'
TRAIN_DATA_FILE=f'{path}{comp}train.csv'
TEST_DATA_FILE=f'{path}{comp}test.csv'<define_variables>
coefficients_ = coefficients.copy()
coefficients_[0] = 1.65
train_predictions = optR.predict(oof_train, coefficients_ ).astype(np.int8)
 test_predictions = optR.predict(oof_test.mean(axis=1), coefficients_ ).astype(np.int8 )
<load_from_csv><EOS> 
submission = pd.DataFrame({'PetID': test['PetID'].values, 'AdoptionSpeed': test_predictions})
 submission.to_csv('submission.csv', index=False )