Kaggle competition: Avaliação de Carros
<SOS> metric: log_loss Kaggle data source: opt-car-2019-1<import_modules>
print(os.listdir(".. /input"))
print(tf.__version__ )<load_from_csv>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv('.. /input/test.csv')
 test.head(5 )
train_data = pd.read_csv('.. /input/train.csv')
 test_data = pd.read_csv('.. /input/test.csv' )<data_type_conversions>
train.isnull().any()
train_data = train_data.values
test_data = test_data.values<normalization>
test.isnull().any()
np.random.shuffle(train_data )<feature_engineering>
test_ids = test['Id']
test = test.drop(['Id'], 1 )
train_digits = train_digits / 255.0
val_digits = val_digits / 255.0
test_digits = test_digits / 255.0<define_search_model>
le = preprocessing.LabelEncoder()
le.fit(train['class'])
 print(list(le.classes_))
train['class'] = le.transform(train['class'] )
X0 = Input(shape =(28,28,1))
X = Conv2D(filters = 32, kernel_size = 3, padding = 'Same', activation ='relu' )(X0)
 X = Conv2D(filters = 32, kernel_size = 3, padding = 'Same', activation ='relu' )(X)
 X = MaxPooling2D(pool_size = 2, strides = 2 )(X)
 X = Dropout(0.25 )(X)
 X = Conv2D(filters = 64, kernel_size = 3, padding = 'Same', activation ='relu' )(X)
 X = Conv2D(filters = 64, kernel_size = 3, padding = 'Same', activation ='relu' )(X)
 X = MaxPooling2D(pool_size = 2, strides = 2 )(X)
 X = Dropout(0.25 )(X)
 X = Flatten()(X)
 X = Dense(512, activation = "relu" )(X)
 X = Dropout(0.50 )(X)
 Out = Dense(10, activation = "softmax" )(X )<load_pretrained>
train_ids = train['Id']
train = train.drop(['Id'], 1 )
model = Model(inputs=X0, outputs=Out)
 model.compile(optimizer = 'adam',
loss = 'sparse_categorical_crossentropy',
metrics=['accuracy'])
 initial_model_weights = model.get_weights()<train_model>
x_data = train.drop('class',axis=1)
 y_labels = train['class']
X_train, X_test, y_train, y_test = train_test_split(x_data, y_labels, test_size=0.3, random_state=101 )
history = model.fit(train_digits, train_labels,
epochs=10,
verbose=2,
validation_data=(val_digits,val_labels))<save_model>
buying = tf.feature_column.categorical_column_with_vocabulary_list("buying", ['high', 'low', 'med', 'vhigh'])
 maintainence = tf.feature_column.categorical_column_with_vocabulary_list("maintainence", ['high', 'low', 'med', 'vhigh'])
 doors = tf.feature_column.categorical_column_with_vocabulary_list("doors", ['3', '4', '5more', '2'])
 persons = tf.feature_column.categorical_column_with_vocabulary_list("persons", ['4', 'more', '2'])
 lug_boot = tf.feature_column.categorical_column_with_vocabulary_list("lug_boot", ['small', 'med', 'big'])
 safety = tf.feature_column.categorical_column_with_vocabulary_list("safety", ['low', 'med', 'high'] )
model.save("cnn1.h5" )<save_to_csv>
feat_cols = [buying, maintainence, doors, persons, lug_boot, safety]
predictions = model.predict(test_digits)
 predictions = np.argmax(predictions,axis = 1)
 predictions = pd.Series(predictions,name = 'Label')
 ids = pd.Series(range(1,28001),name = 'ImageId')
 predictions = pd.concat([predictions,ids],axis = 1)
 predictions.to_csv('pred1.csv',index = False)
 del(predictions)
 del(ids )<load_pretrained>
input_func = tf.estimator.inputs.pandas_input_fn(x=X_train,
y=y_train,
batch_size=500,
num_epochs=10000,
shuffle=True )
model.set_weights(initial_model_weights )<define_variables>
input_func_test = tf.estimator.inputs.pandas_input_fn(x=test,
num_epochs=500,
shuffle=False )
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
width_shift_range=0.1,
height_shift_range=0.1,
shear_range=0.1,
zoom_range=0.1,
channel_shift_range=0.,
fill_mode='nearest',
cval=0.,
horizontal_flip=False,
vertical_flip=False,
rescale=None,
preprocessing_function=None,
data_format=None,
validation_split=0.0)
 datagen.fit(train_digits )<train_model>
model = tf.estimator.LinearClassifier(feature_columns=feat_cols, n_classes=4 )
batch_size = 64
gen_history = model.fit_generator(datagen.flow(train_digits, train_labels, batch_size=batch_size),
epochs=10,
verbose=2,
steps_per_epoch=train_digits.shape[0]/batch_size,
validation_data=(val_digits,val_labels))
plot_history(gen_history.history )<save_to_csv>
model.train(input_fn=input_func,steps=5000 )
predictions = model.predict(test_digits)
 predictions = np.argmax(predictions,axis = 1)
 predictions = pd.Series(predictions,name = 'Label')
 ids = pd.Series(range(1,28001),name = 'ImageId')
 predictions = pd.concat([predictions,ids],axis = 1)
 predictions.to_csv('pred2.csv',index = False)
 del(predictions)
 del(ids)
 model.save("cnn2.h5" )<load_from_csv>
pred_fn = tf.estimator.inputs.pandas_input_fn(x=X_test, batch_size=len(X_test), shuffle=False )
train_df=pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 test_df=pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
 def data_info(d):
print('number of variables: ',d.shape[1])
 print('number of tweets: ',d.shape[0])
 print('variables names: ')
 print(d.columns)
 print('variables data-types: ')
 print(d.dtypes)
 print('missing values: ')
 c=d.isnull().sum()
print(c[c>0] )<filter>
predictions = list(model.predict(input_fn=pred_fn))
probs = pd.Series([pred['class_ids'][0] for pred in predictions] )
train_df[train_df['target']==1].loc[:4,'text']<string_transform>
final_preds = []
for pred in predictions:
final_preds.append(pred['class_ids'][0])
 print(classification_report(y_test,final_preds))
train_disasters={'earthquake':0,'fire':0,'fires':0,'shelter':0}
test_disasters={'earthquake':0,'fire':0,'fires':0,'shelter':0}
for i,j in zip(train_df['text'],test_df['text']):
if 'earthquake' in i.split() :
train_disasters['earthquake']+=1
elif 'earthquake' in j.split() :
test_disasters['earthquake']+=1
if 'fire' in i.split() :
train_disasters['fire']+=1
elif 'fire' in j.split() :
test_disasters['fire']+=1
if 'fires' in i.split() :
train_disasters['fires']+=1
elif 'fires' in j.split() :
test_disasters['fires']+=1
if 'shelter' in i.split() :
train_disasters['shelter']+=1
elif 'shelter' in j.split() :
test_disasters['shelter']+=1<count_values>
eval_input_func = tf.estimator.inputs.pandas_input_fn(x=X_test,y=y_test,batch_size=len(X_test),shuffle=False )
print('number of tweets that fires,fire and earthquacke mentioned in train data: ',train_disasters)
 print('number of tweets that fires,fire and earthquacke mentioned in test data: ',test_disasters )<feature_engineering>
results = model.evaluate(eval_input_func)
 results
def tokenization(text):
lst=text.split()
return lst
train_df['text']=train_df['text'].apply(tokenization)
 test_df['text']=test_df['text'].apply(tokenization )<categorify>
pred_fn_test = tf.estimator.inputs.pandas_input_fn(x=test, batch_size=len(test), shuffle=False )
def lowercasing(lst):
new_lst=[]
for i in lst:
i=i.lower()
new_lst.append(i)
 return new_lst
train_df['text']=train_df['text'].apply(lowercasing)
 test_df['text']=test_df['text'].apply(lowercasing )<feature_engineering>
predictions_test = list(model.predict(input_fn=pred_fn_test))
probs_test = pd.Series([pred['class_ids'][0] for pred in predictions_test] )
def remove_punctuations(lst):
new_lst=[]
for i in lst:
for j in s.punctuation:
i=i.replace(j,'')
 new_lst.append(i)
 return new_lst
train_df['text']=train_df['text'].apply(remove_punctuations)
 test_df['text']=test_df['text'].apply(remove_punctuations )<feature_engineering>
preds_test_1 = []
for pred in predictions_test:
preds_test_1.append(pred['class_ids'][0])
 print(len(preds_test_1))
def remove_numbers(lst):
nodig_lst=[]
new_lst=[]
for i in lst:
for j in s.digits:
i=i.replace(j,'')
 nodig_lst.append(i)
 for i in nodig_lst:
if i!='':
new_lst.append(i)
 return new_lst
train_df['text']=train_df['text'].apply(remove_numbers)
 test_df['text']=test_df['text'].apply(remove_numbers )<feature_engineering>
preds_test = le.inverse_transform(preds_test_1)
 print(type(preds_test))
def remove_stopwords(lst):
stop=stopwords.words('english')
 new_lst=[]
for i in lst:
if i not in stop:
new_lst.append(i)
 return new_lst
train_df['text']=train_df['text'].apply(remove_stopwords)
 test_df['text']=test_df['text'].apply(remove_stopwords )<feature_engineering>
df = pd.DataFrame(columns=['Id', 'Class_vgood', 'Class_good', 'Class_acc', 'Class_unacc'])
 for i, ids, preds in zip(range(len(test_ids)) , test_ids, preds_test):
if(preds == 'vgood'):
submission = pd.DataFrame({
"Id": ids,
"Class_vgood": 1,
"Class_good": 0,
"Class_acc": 0,
"Class_unacc": 0,
}, index=[i])
 df = df.append(submission)
 if(preds == 'good'):
submission = pd.DataFrame({
"Id": ids,
"Class_vgood": 0,
"Class_good": 1,
"Class_acc": 0,
"Class_unacc": 0,
}, index=[i])
 df = df.append(submission)
 if(preds == 'acc'):
submission = pd.DataFrame({
"Id": ids,
"Class_vgood": 0,
"Class_good": 0,
"Class_acc": 1,
"Class_unacc": 0,
}, index=[i])
 df = df.append(submission)
 if(preds == 'unacc'):
submission = pd.DataFrame({
"Id": ids,
"Class_vgood": 0,
"Class_good": 0,
"Class_acc": 0,
"Class_unacc": 1,
}, index=[i])
 df = df.append(submission )
def remove_spaces(lst):
new_lst=[]
for i in lst:
i=i.strip()
new_lst.append(i)
 return new_lst
train_df['text']=train_df['text'].apply(remove_spaces)
 test_df['text']=test_df['text'].apply(remove_spaces )<feature_engineering>
df.to_csv('sampleSubmission.csv', index=False )