Kaggle competition: car-classification
<SOS> metric: categorizationaccuracy Kaggle data source: car-classification<data_type_conversions>
!pip show fastai
data['text_cleaned'] = data['text_cleaned'].apply(lambda s : s.lower())
 test['text_cleaned'] = test['text_cleaned'].apply(lambda s : s.lower() )<install_modules>
%matplotlib inline
!pip install flair<create_dataframe>
MODEL=resnet50
BATCH = 64
SIZE = 320
DATA_PATH = '.. /input/'
PATH = ".. /car-classification/"
df = data[['target_relabeled', 'text_cleaned']]<normalization>
def accuracy(preds, targs):
preds = torch.max(preds, dim=1)[1]
return(preds==targs ).float().mean()
df = shuffle(df )<save_to_csv>
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/sample.csv")
 train_df.head()
df.iloc[0:int(len(df)*0.8)].to_csv('data/train.csv', sep='\t', index = False, header = False)
 df.iloc[int(len(df)*0.8):int(len(df)*0.9)].to_csv('data/test.csv', sep='\t', index = False, header = False)
 df.iloc[int(len(df)*0.9):].to_csv('data/dev.csv', sep='\t', index = False, header = False);<choose_model_class>
train_df.Category.value_counts()
data_folder = 'data/'
column_name_map = {1: "text", 0: "label_topic"}
corpus: Corpus = CSVClassificationCorpus(data_folder,
column_name_map,
skip_header=True,
delimiter='\t',
)<choose_model_class>
image = Image.open(DATA_PATH+'/train/train/0/100380.jpg')
 imgplot = plt.imshow(image)
 plt.show()
image.size
word_embeddings = WordEmbeddings('glove')
 bert_embedding = TransformerWordEmbeddings('bert-base-multilingual-cased' )<choose_model_class>
tfms = get_transforms(max_zoom=1., max_warp=0.2, max_lighting=0.3,
xtra_tfms=[cutout(n_holes=(1,20)) ]
)
document_embeddings = DocumentRNNEmbeddings([word_embeddings, bert_embedding], rnn_type='LSTM', hidden_size=256, rnn_layers=2 )<choose_model_class>
data = ImageDataBunch.from_folder(DATA_PATH, train=PATH+'train/train/', test='test/test_upload/',
ds_tfms=tfms, padding_mode='zeros',
valid_pct=0.1, size=SIZE,
classes=['0','1','2','3','4','5','6','7','8','9'],
bs=BATCH, num_workers=0 ).normalize(imagenet_stats)
 data.path = pathlib.Path(PATH )
label_dict = corpus.make_label_dictionary()
classifier = TextClassifier(document_embeddings, label_dictionary=label_dict )<train_model>
data.show_batch(rows=4, figsize=(12,9))
trainer = ModelTrainer(classifier, corpus)
 path = "model/"
history = trainer.train('model/',
learning_rate=0.1,
mini_batch_size=64,
anneal_factor=0.5,
patience=2,
max_epochs=30 )<predict_on_test>
learn = cnn_learner(data, MODEL, metrics=accuracy, model_dir=PATH )
sentence = Sentence(test.text_cleaned[0])
 classifier.predict(sentence)
 print(sentence.labels[0] )<predict_on_test>
learn.fit_one_cycle(5, slice(5e-3))
classifier.predict([sentence, sentence, sentence, sentence, sentence] )<predict_on_test>
accuracy(*learn.get_preds() )
def predict(text, model=None):
sentence = Sentence(text)
 if model:
model.predict(sentence)
 else:
classifier.predict(sentence)
 return sentence.labels[0].value<predict_on_test>
learn.save('stage-1-resnet50' )
predict(test.text_cleaned[10] )<predict_on_test>
learn.fit_one_cycle(5, max_lr=slice(1e-7,1e-5),)
 learn.recorder.plot()
learn.recorder.plot_losses()
learn.recorder.plot_metrics()
test_pred = list()
for text in tqdm(test.text_cleaned):
out = predict(text)
 test_pred.append(out )<save_to_csv>
accuracy(*learn.get_preds() )
submission['target'] = test_pred
submission.to_csv('submission.csv', index=False )<import_modules>
learn.save('stage-2-resnet50', return_path=True )
<install_modules>
learn = cnn_learner(data, MODEL, metrics=accuracy, model_dir=PATH)
 learn.unfreeze()
!pip install pyenchant
! apt-get update
! apt-get install libenchant-dev -y<import_modules>
learn.fit_one_cycle(30, max_lr=1e-4)
 learn.recorder.plot()
learn.recorder.plot_losses()
learn.recorder.plot_metrics()
import numpy as np
import pandas as pd
import time
from datetime import datetime
from sklearn.model_selection import train_test_split
from google.cloud import storage
from google.cloud import automl_v1beta1 as automl
from automlwrapper import AutoMLWrapper
from fastai.text import *
import spacy
import pandas as pd
import numpy as np
import re
import nltk
from nltk import word_tokenize
from tqdm import tqdm
import enchant
from nltk.metrics import edit_distance<categorify>
accuracy(*learn.get_preds() )
replacement_patterns = [ 
(r'won't', 'will not'), 
(r'can't', 'cannot'), 
(r'i'm', 'i am'), 
(r'ain't', 'is not'), 
(r'(\w+)'ll', '\g<1> will'), 
(r'(\w+)n't', '\g<1> not'), 
(r'(\w+)'ve', '\g<1> have'), 
(r'(\w+)'s', '\g<1> is'), 
(r'(\w+)'re', '\g<1> are'), 
(r'(\w+)'d', '\g<1> would'),
]
class RegexpReplacer(object):
def __init__(self, patterns=replacement_patterns):
self.patterns = [(re.compile(regex), repl)for(regex, repl)in patterns]
def replace(self, text):
s = text
for(pattern, repl)in self.patterns:
s = re.sub(pattern, repl, s)
 return s
class SpellingReplacer(object):

 def __init__(self, dict_name='en', max_dist=2):
self.spell_dict = enchant.Dict(dict_name)
 self.max_dist = max_dist
def replace(self, word):
if self.spell_dict.check(word):
return word
suggestions = self.spell_dict.suggest(word)
 if suggestions and edit_distance(word, suggestions[0])<= self.max_dist:
return suggestions[0]
else:
return word<define_variables>
learn.freeze()
learn.save('stage-3-resnet50', return_path=True )
input_path = '/kaggle/input/'
output_path = '/kaggle/working/'<load_from_csv>
submission = pd.read_csv('.. /input/sample.csv')
 submission.shape
ids_with_target_error = [328,443,513,2619,3640,3900,4342,5781,6552,6554,6570,6701,6702,6729,6861,7226]
train_orig = pd.read_csv(input_path+'nlp-getting-started/train.csv', encoding = 'latin-1')
 incorrect = train_orig[train_orig['id'].isin(ids_with_target_error)]<load_from_csv>
data.train_ds.x
df_train = pd.read_csv(input_path+'disasters-on-social-media/socialmedia-disaster-tweets-DFE.csv', encoding = 'latin-1')
 df_train = df_train[['_unit_id','keyword','location','text','choose_one']]
incorrect_ids = incorrect.merge(df_train,on=['text'])['_unit_id']
df_train.rename(columns = {'_unit_id': 'id'}, inplace = True)
 df_train['target'] = df_train['choose_one'].map({'Relevant': 1, 'Not Relevant': 0})
 df_train.drop(['choose_one'], axis = 1, inplace = True)
 df_train = df_train[-df_train['target'].isna() ]
df_train = df_train.astype({'target': 'int64'})
 df_train.loc[df_train['id'].isin(incorrect_ids),'target'] = 0
df_train.head()<load_from_csv>
data.test_ds.x
df_test = pd.read_csv(input_path+'nlp-getting-started/test.csv')
 df_test.head()<load_from_csv>
preds_test, _ = learn.get_preds(DatasetType.Test )
cols = ['target','id','date','flag','user','text']
train_add = pd.read_csv(input_path+'disastertweetsinput/train_clean_add.csv',names=cols,encoding = 'latin-1', skiprows=1)
 train_add.head()<concatenate>
preds_test_s = pd.DataFrame(preds_test)
 preds_test_s.to_csv('preds_sub.csv', index=False)
 preds_test_s.shape
train_add = df_train[['id','text','target']].append(train_add[['id','text','target']] )<feature_engineering>
preds_test = np.argmax(preds_test, axis=1)
 preds_test = preds_test.numpy()
def clean_tweet(text):
text = re.sub(r'http\S+', '', text)
 replacer = RegexpReplacer()
text = replacer.replace(text)
 text = re.sub(r'\b', ' ', text)
 text = re.sub(r'-', ' ', text)
 tokenizer = nltk.RegexpTokenizer(r"\w+")
 tokens = tokenizer.tokenize(text)
 replacer = SpellingReplacer()
tokens = [replacer.replace(t)for t in tokens]
wnl = nltk.WordNetLemmatizer()
tokens = [wnl.lemmatize(t)for t in tokens]
porter = nltk.PorterStemmer()
tokens = [porter.stem(t)for t in tokens]
text = ' '.join(tokens)
 return(text )<feature_engineering>
fnames = [f.name for f in learn.data.test_ds.items]
submission = pd.DataFrame({'Id':fnames, 'Category':preds_test}, columns=['Id', 'Category'] )
tweets = df_test['text']
tweets_cleaned = tweets.progress_apply(clean_tweet)
 df_test['text_clean'] = tweets_cleaned<remove_duplicates>
submission.to_csv('submission_v23.csv', index=False )
df = df_test[['text_clean']].append(df_train[['text_clean']])
 df = df.drop_duplicates().reset_index() ['text_clean']
df.head()<string_transform>
preds_test_tta, _ = learn.TTA(ds_type=DatasetType.Test)
train = []
tokenizer = Tokenizer()
tok = SpacyTokenizer('en')
 for line in tqdm(df_train.text_clean):
lne = ' '.join(tokenizer.process_text(line, tok))
train.append(lne)
 df_train['text_tokens'] = train
test = []
tokenizer = Tokenizer()
tok = SpacyTokenizer('en')
 for line in tqdm(df_test.text_clean):
lne = ' '.join(tokenizer.process_text(line, tok))
test.append(lne)
 df_test['text_tokens'] = test<define_variables>
preds_test_tta_s = pd.DataFrame(preds_test_tta)
 preds_test_tta_s.to_csv('preds_tta_sub.csv', index=False)
 preds_test_tta_s.shape
PROJECT_ID = 'kaggle-tweets-0234'
bucket_name = 'kaggle-tweets-0234-lcm'
region = 'us-central1'
dataset_display_name = 'disaster_tweets'
model_display_name = 'disaster_tweets_model1'
storage_client = storage.Client(project=PROJECT_ID)
 client = automl.AutoMlClient()<train_model>
preds_test_tta = np.argmax(preds_test_tta, axis=1)
 preds_test_tta = preds_test_tta.numpy()
project_location = client.location_path(PROJECT_ID, 'us-central1')
 response = client.list_models(project_location, '')
 print('List of models:')
 for model in response:
if model.deployment_state == \
automl.enums.Model.DeploymentState.DEPLOYED:
deployment_state = 'deployed'
else:
deployment_state = 'undeployed'
print(u'Model name: {}'.format(model.name))
print(u'Model id: {}'.format(model.name.split('/')[-1]))
print(u'Model display name: {}'.format(model.display_name))
print(u'Model create time:')
 print(u'\tseconds: {}'.format(model.create_time.seconds))
print(u'\tnanos: {}'.format(model.create_time.nanos))
print(u'Model deployment state: {}'.format(deployment_state))<load_from_csv>
fnames = [f.name for f in learn.data.test_ds.items]
submission = pd.DataFrame({'Id':fnames, 'Category':preds_test_tta}, columns=['Id', 'Category'] )
nlp_train_df = df_train
nlp_test_df = df_test
def callback(operation_future):
result = operation_future.result()<load_pretrained>
submission['Category'] = preds_test_tta
submission.to_csv('tta_submission_v23.csv', index=False )
<prepare_output><EOS> 
shutil.rmtree(PATH )