Kaggle competition: kaggle18011884
count_vect = CountVectorizer()
X_train_counts = count_vect.fit_transform(df_train.ingredients_concat)
 X_test_counts = count_vect.transform(df_test.ingredients_concat)
 print('A total of', X_train_counts.shape[0], 'recipes')
 print('A total of', X_train_counts.shape[1], 'unique ingredient words' )<create_dataframe>
train=pd.read_csv('.. /input/kaggle18011884/train_cabbage_price.csv')
 train.avgPrice.plot()
ingredient_names = np.array(count_vect.get_feature_names())
 ingredient_counts = dict(zip(ingredient_names, X_train_counts.sum(axis=0 ).tolist() [0]))
ingredient_counts = pd.DataFrame(list(ingredient_counts.items()), columns=['ingredients', 'counts'])
 ingredient_counts['counts_frac'] = ingredient_counts['counts'].apply(lambda x: x/X_train_counts.shape[0] )<sort_values>
test=pd.read_csv('.. /input/kaggle18011884/test_cabbage_price.csv')
 test
ingredient_counts.sort_values(by='counts', ascending=False ).head(n=10 )<categorify>
train['yeargb']=[y[:6] for y in train['year'].astype('str')]
test['yeargb']=test['year']
train['month']=[np.int(y[4:6])for y in train['year'].astype('str')]
test['month']=[np.int(y[-2:])for y in test['year'].astype('str')]
train['year2']=[np.int(y[:4])for y in train['year'].astype('str')]
test['year2']=[np.int(y[:4])for y in test['year'].astype('str')]
traingb=train.groupby('yeargb' ).mean().reset_index()
sns.heatmap(traingb.corr() )
tfidf = TfidfTransformer()
X_train_tfidf = tfidf.fit_transform(X_train_counts)
 X_test_tfidf = tfidf.transform(X_test_counts)
 print(X_train_tfidf.shape )<categorify>
traingb.reset_index().append(test.reset_index() )
mlb = preprocessing.MultiLabelBinarizer()
y_train_mlb = mlb.fit_transform(df_train.cuisine.values.reshape(-1,1))
le = preprocessing.LabelEncoder()
y_train_le = le.fit_transform(df_train.cuisine )<create_dataframe>
pd.DataFrame(np.dot(np.array(ymean.avgPrice ).reshape(-1,1), np.array(mmean.avgPrice ).reshape(1,-1)))/(traingb.avgPrice.mean() **2 )
<sort_values><EOS> 
def clustertechniques2(dtrain,label,indexv):
print('
cols=[ci for ci in dtrain.columns if ci not in [indexv,'index',label]]
dtest=dtrain[dtrain[label].isnull() ==True][[indexv,label]]
print(dtest)
 print('encodings after shape',dtrain.shape)
 X_train=dtrain[dtrain[label].isnull() ==False].drop([indexv,label],axis=1 ).fillna(0)
 Y_train=dtrain[dtrain[label].isnull() ==False][label]
X_test=dtrain[dtrain[label].isnull() ==True].drop([indexv,label],axis=1 ).fillna(0)
 Y_test=dtrain[dtrain[label].isnull() ==True][label].fillna(0)
 print(Y_test)
 for xi in range(len(Y_test)) :
Y_test.iloc[xi]=np.random.random(( 1,1)) [0]
print(Y_test)
 if len(X_test)==0:
X_train,X_test,Y_train,Y_test = train_test_split(dtrain.drop(label,axis=1 ).fillna(0),dtrain[label],test_size=0.25,random_state=0)
 lenxtr=len(X_train)
 print('splitting data train test X-y',X_train.shape,Y_train.shape,X_test.shape,Y_test.shape)
 scale = preprocessing.MinMaxScaler().fit(X_train)
 X_train = scale.transform(X_train)
 X_test = scale.transform(X_test)
 clusters = [Dummy(1),
PCA(n_components=0.7,random_state=0,whiten=True),
FastICA(n_components=7,random_state=0),
TruncatedSVD(n_components=5, n_iter=7, random_state=42),
NMF(n_components=10,random_state=0),
UMAP(n_neighbors=5,n_components=10, min_dist=0.3,metric='minkowski'),
TSNE(n_components=2,random_state=0)
 ]
clunaam=["raw",'PCA','tSVD','UMAP','tSNE']
classifiers = [
ExtraTreesRegressor() ,
RandomForestRegressor(random_state=1, n_estimators=10),
BayesianRidge() ,
RANSACRegressor() ,
KNeighborsRegressor() ,
ElasticNetCV(cv=5, random_state=0),
HuberRegressor() ,
LinearRegression() ,
]
clanaam= ['xTreer','rFor','BaysR','Ransac','KNNr','elast','huber','linear',]
results=[]
for clu in clusters:
clunm=clunaam[clusters.index(clu)]
X_total_clu = clu.fit_transform(np.concatenate(( X_train,X_test),axis=0))
X_total_clu=np.concatenate(( X_total_clu,np.concatenate(( X_train,X_test),axis=0)) ,axis=1)
 print(X_total_clu.shape)
 plt.scatter(X_total_clu[:lenxtr,0],X_total_clu[:lenxtr,1],c=Y_train.values,cmap='prism')
 plt.title(clu)
 plt.show()
for cla in classifiers:
start = datetime.datetime.now()
clanm=clanaam[classifiers.index(cla)]
print(' ',cla)
 cla.fit(X_total_clu,np.concatenate(( Y_train,Y_test)))
 cla.fit(X_total_clu[:lenxtr],Y_train)
 trainpredi=cla.predict(X_total_clu[:lenxtr])
 testpredi=cla.predict(X_total_clu[lenxtr:])
 if classifiers.index(cla)in [0,2,3,4,5,7,8,9,10,11,12,13]:
trainprediprob=cla.predict(X_total_clu[:lenxtr])
 testprediprob=cla.predict(X_total_clu[lenxtr:])
 plt.scatter(x=testprediprob, y=testpredi, marker='.', alpha=0.53)
 plt.show()
if len(dtest)==0:
test_score=cla.score(X_total_clu[lenxtr:],Y_test)
 mse = mean_squared_error(testpredi,Y_test)
 train_score=cla.score(X_total_clu[:lenxtr],Y_train)
 li = [clunm,clanm,train_score,test_score,mse]
results.append(li)
 r2s=r2_score(testpredi,Y_test)
 print(r2s)
 plt.title(clanm+'test corr & mse:'+np.str(test_score)+' '+np.str(mse)+' and test confusionmatrix')
 plt.scatter(x=Y_test, y=testpredi, marker='.', alpha=1)
 plt.scatter(x=[np.mean(Y_test)], y=[np.mean(testpredi)], marker='o', color='red')
 plt.xlabel('Real test'); plt.ylabel('Pred.test')
 plt.show()
else:
testpredlabel=testpredi
print('train correl',r2_score(trainpredi,Y_train),'mse ',mean_squared_error(trainpredi,Y_train))
submit = pd.DataFrame({'Id': dtest[indexv],'Expected': testpredlabel})
 submit['Expected']=submit['Expected'].astype('int')
 filenaam='subm_'+clunm+'_'+clanm+'.csv'
submit.to_csv(path_or_buf =filenaam, index=False)
 print(clanm,'0 classifier time',datetime.datetime.now() -start)
 if len(dtest)==0:
print(pd.DataFrame(results ).sort_values(3))
submit=[]
return submit
class Dummy() :
def __init__(self, feature_names):
self._feature_names = feature_names
def fit(self, X, y = None):
return self
def fit_transform(self, X, y = None):
return X
clustertechniques2(traingb.reset_index().append(test.reset_index() ).drop('year',axis=1),'avgPrice','index' )