Kaggle competition: Technidus machine learning competition 2
contraction_mapping = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have", "couldn't": "could not", "didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not", "he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you", "how'll": "how will", "how's": "how is", "I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have","I'm": "I am", "I've": "I have", "i'd": "i would", "i'd've": "i would have", "i'll": "i will", "i'll've": "i will have","i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would", "it'd've": "it would have", "it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us", "ma'am": "madam", "mayn't": "may not", "might've": "might have","mightn't": "might not","mightn't've": "might not have", "must've": "must have", "mustn't": "must not", "mustn't've": "must not have", "needn't": "need not", "needn't've": "need not have","o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not", "sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would", "she'd've": "she would have", "she'll": "she will", "she'll've": "she will have", "she's": "she is", "should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have","so's": "so as", "this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is", "there'd": "there would", "there'd've": "there would have", "there's": "there is", "here's": "here is","they'd": "they would", "they'd've": "they would have", "they'll": "they will", "they'll've": "they will have", "they're": "they are", "they've": "they have", "to've": "to have", "wasn't": "was not", "we'd": "we would", "we'd've": "we would have", "we'll": "we will", "we'll've": "we will have", "we're": "we are", "we've": "we have", "weren't": "were not", "what'll": "what will", "what'll've": "what will have", "what're": "what are", "what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did", "where's": "where is", "where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have", "why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not", "won't've": "will not have", "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have", "y'all": "you all", "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have","you'd": "you would", "you'd've": "you would have", "you'll": "you will", "you'll've": "you will have", "you're": "you are", "you've": "you have" }
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
mispell_dict = {'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', "mastrubating": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', "whst": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization'}<string_transform>
def Education_to_numeric(bp):
if bp == 'Bachelors ':
return 4
if bp == 'Partial College':
return 3
if bp == 'High School':
return 2
if bp == 'Graduate Degree':
return 1
if bp == 'Partial High School':
return 0
edu=train_data.Education
edut=test_data.Education
edupp = edu.apply(Education_to_numeric)
 edud = edut.apply(Education_to_numeric)
 train_data.Education=edupp
test_data.Education=edud
def occupation_to_numeric(b):
if b == 'Clerical':
return 1
if b == 'Professional':
return 1
if b == 'Manual':
return 0
if b == 'Skilled Manual':
return 0
if b == 'Management':
return 1
p=train_data.Occupation
pt=test_data.Occupation
d = p.apply(occupation_to_numeric)
 dt = pt.apply(occupation_to_numeric)
 train_data.Occupation=d
test_data.Occupation=dt
def COUNTRY_to_numeric(d):
if d == 'Canada':
return 3
if d == 'France':
return 4
if d == 'Australia':
return 1
if d == 'United Kingdom':
return 2
if d == 'United States':
return 5
if d == 'Germany':
return 6
pon=train_data.CountryRegionName
ptu=test_data.CountryRegionName
qa = pon.apply(COUNTRY_to_numeric)
 qaw= ptu.apply(COUNTRY_to_numeric)
 train_data.CountryRegionName=qa
test_data.CountryRegionName=qaw
def marital_to_numeric(ip):
if ip == 'M':
return 0
if ip == 'S':
return 1
pl= train_data.MaritalStatus
plt=test_data.MaritalStatus
ml = pl.apply(marital_to_numeric)
 mlt = plt.apply(marital_to_numeric)
 train_data.MaritalStatus=ml
test_data.MaritalStatus=mlt
def region_to_numeric(a):
if a == 'M':
return 2
if a == 'F':
return 1
z=train_data.Gender
zt=test_data.Gender
c = z.apply(region_to_numeric)
 ct = zt.apply(region_to_numeric)
 train_data.Gender=c
test_data.Gender=ct
yy=train_data.BirthDate
now=pd.Timestamp(DT.datetime.now())
 nuu = pd.to_datetime(yy)
 nuu=(now-nuu)/365
ny=nuu.dt.days
train_data.BirthDate=ny
yyt=test_data.BirthDate
nowt=pd.Timestamp(DT.datetime.now())
 nuut = pd.to_datetime(yyt)
 nuut=(nowt-nuut)/365
nyt=nuut.dt.days
test_data.BirthDate=nyt
train_data.PostalCode=train_data.TotalChildren.subtract(train_data.NumberChildrenAtHome, fill_value=0)
 test_data.PostalCode=test_data.TotalChildren.subtract(test_data.NumberChildrenAtHome, fill_value=0)
 gh=train_data.YearlyIncome-train_data.TotalChildren
def age_to_numeric(qa):
if qa <=37:
return 3
if 51>qa >=44:
return 4
if 58>qa >=51:
return 1
if 65>qa >=58:
return 2
if qa >=65:
return 5
train_data.BirthDate=100/train_data.BirthDate**3.9
test_data.BirthDate=100/test_data.BirthDate**3.9
def ave_to_numeric(qa):
if qa <40:
return 1
if 50>qa >=40:
return 2
if 59>qa >=50:
return 3
if 65>qa >=59:
return 4
if qa >=65:
return 5
train_data.AveMonthSpend=(np.log(train_data.AveMonthSpend)) *12
test_data.AveMonthSpend=(np.log(test_data.AveMonthSpend)) *12
train_data.Suffix=(train_data.YearlyIncome**3.3)-(train_data.AveMonthSpend**3)
 test_data.Suffix=(test_data.YearlyIncome**3.3)-(test_data.AveMonthSpend**3)
 train_data.YearlyIncome=(train_data.YearlyIncome**3.3)
 test_data.YearlyIncome=(test_data.YearlyIncome**3.3)
 train_data.TotalChildren=(train_data.TotalChildren*2)
 test_data.TotalChildren=(test_data.TotalChildren*2)
 train_data.City=train_data.AveMonthSpend*30
test_data.City=test_data.AveMonthSpend*30
train_data.PostalCode=train_data.TotalChildren.subtract(train_data.NumberChildrenAtHome, fill_value=0)
 test_data.PostalCode=test_data.TotalChildren.subtract(test_data.NumberChildrenAtHome, fill_value=0)
 train_data.PostalCode=1/train_data.CustomerID**1/2
test_data.PostalCode=1/test_data.CustomerID**1/2
y = train_data.BikeBuyer
features = ['TotalChildren','YearlyIncome',
'HomeOwnerFlag','NumberChildrenAtHome','Gender','MaritalStatus','BirthDate','CountryRegionName',
'Occupation','PostalCode','Suffix','Education','PostalCode'
]
train_data.AveMonthSpend.plot.hist(color='blue',bins=50)
 plt.show()
X= train_data[features]
textX=test_data[features]
corr= train_data.corr()
f, ax = plt.subplots(figsize=(10, 5))
sns.heatmap(corr,cmap='coolwarm',linewidths=2.0, annot=True)
 my_imputer = SimpleImputer()
X = my_imputer.fit_transform(X)
 textX = my_imputer.fit_transform(textX)
def known_contractions(embed):
known = []
for contract in contraction_mapping:
if contract in embed:
known.append(contract)
 return known
def clean_contractions(text, mapping):
specials = ["’", "‘", "´", "`"]
for s in specials:
text = text.replace(s, "'")
 text = ' '.join([mapping[t] if t in mapping else t for t in text.split(" ")])
 return text
def correct_spelling(x, dic):
for word in dic.keys() :
x = x.replace(word, dic[word])
 return x
def unknown_punct(embed, punct):
unknown = ''
for p in punct:
if p not in embed:
unknown += p
unknown += ' '
return unknown
def clean_numbers(x):
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
return x
def clean_special_chars(text, punct, mapping):
for p in mapping:
text = text.replace(p, mapping[p])
 for p in punct:
text = text.replace(p, f' {p} ')
 specials = {'\u200b': ' ', '…': '...', '\ufeff': '', 'करना': '', 'है': ''}
for s in specials:
text = text.replace(s, specials[s])
 return text
def add_lower(embedding, vocab):
count = 0
for word in vocab:
if word in embedding and word.lower() not in embedding:
embedding[word.lower() ] = embedding[word]
count += 1
print(f"Added {count} words to embedding")
 def clean_text(x):
x = str(x)
 for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x
def clean_numbers(x):
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
return x
def _get_mispell(mispell_dict):
mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
return mispell_dict, mispell_re
mispellings, mispellings_re = _get_mispell(mispell_dict)
 def replace_typical_misspell(text):
def replace(match):
return mispellings[match.group(0)]
return mispellings_re.sub(replace, text)
 def add_features(df):
df['question_text'] = df['question_text'].progress_apply(lambda x:str(x))
df['total_length'] = df['question_text'].progress_apply(len)
 df['capitals'] = df['question_text'].progress_apply(lambda comment: sum(1 for c in comment if c.isupper()))
df['caps_vs_length'] = df.progress_apply(lambda row: float(row['capitals'])/float(row['total_length']), axis=1)
 df['num_words'] = df.question_text.str.count('\S+')
 df['num_unique_words'] = df['question_text'].progress_apply(lambda comment: len(set(w for w in comment.split())))
 df['words_vs_unique'] = df['num_unique_words'] / df['num_words']
df['question_text'] = df['question_text'].progress_apply(lambda x: x.lower())
 df['question_text'] = df['question_text'].progress_apply(clean_text)
 df['question_text'] = df['question_text'].progress_apply(clean_numbers)
 df['question_text'] = df['question_text'].progress_apply(replace_typical_misspell)
 return df<normalization>
train_X, val_X, train_y, val_y = train_test_split(X, y, random_state=1)
 xgb = XGBClassifier(n_estimators=380, learning_rate=0.049, random_state=1,min_child_weight=6)
 training_start = time.perf_counter()
xgb.fit(train_X, train_y)
 training_end = time.perf_counter()
prediction_start = time.perf_counter()
preds = xgb.predict(val_X)
 prediction_end = time.perf_counter()
acc_xgb =(preds == val_y ).sum().astype(float)/ len(preds)*100
xgb_train_time = training_end-training_start
xgb_prediction_time = prediction_end-prediction_start
print("XGBoost's prediction accuracy is: %3.2f" %(acc_xgb))
print("Time consumed for training: %4.3f" %(xgb_train_time))
print("Time consumed for prediction: %6.5f seconds" %(xgb_prediction_time))
pre=xgb.predict(textX)
 final=pre.astype(int)
 output = pd.DataFrame({'CustomerID': test_data.CustomerID,
'BikeBuyer': final})
 output.to_csv('Sample Submission file clf.csv', index=False )