Kaggle competition: Kannada MNIST
leak = pd.read_csv(".. /input/disasters-on-social-media/socialmedia-disaster-tweets-DFE.csv")
 leak['target'] =(leak['choose_one']=='Relevant' ).astype(int)
 leak['id'] = leak.index
leak = leak[['id', 'target','text']]
merged_df = pd.merge(test, leak, on='id')
 sub1 = merged_df[['id', 'target']]
sub1.to_csv('submit_1.csv', index=False )<string_transform>
warnings.filterwarnings('ignore' )
test_str = train.loc[417, 'text']
def clean_text(text):
text = re.sub(r'https?://\S+', '', text)
 text = re.sub(r'
',' ', text)
 text = re.sub('\s+', ' ', text ).strip()
return text
print("Original text: " + test_str)
 print("Cleaned text: " + clean_text(test_str))<feature_engineering>
train_data = pd.read_csv('/kaggle/input/Kannada-MNIST/train.csv')
 test_data = pd.read_csv('/kaggle/input/Kannada-MNIST/test.csv')
 y_train = train_data['label']
X = train_data.drop(['label'],axis=1)
 del train_data
def find_hashtags(tweet):
return " ".join([match.group(0)[1:] for match in re.finditer(r"
def find_mentions(tweet):
return " ".join([match.group(0)[1:] for match in re.finditer(r"@\w+", tweet)])or 'no'
def find_links(tweet):
return " ".join([match.group(0)[1:] for match in re.finditer(r"https?://\S+", tweet)])or 'no'
def process_text(df):
df['text_clean'] = df['text'].apply(lambda x: clean_text(x))
df['hashtags'] = df['text'].apply(lambda x: find_hashtags(x))
df['mentions'] = df['text'].apply(lambda x: find_mentions(x))
df['links'] = df['text'].apply(lambda x: find_links(x))
return df
train = process_text(train)
 test = process_text(test )<feature_engineering>
Id = test_data['id']
def create_stat(df):
df['text_len'] = df['text_clean'].apply(len)
 df['word_count'] = df["text_clean"].apply(lambda x: len(str(x ).split()))
df['stop_word_count'] = df['text_clean'].apply(lambda x: len([w for w in str(x ).lower().split() if w in STOPWORDS]))
df['punctuation_count'] = df['text_clean'].apply(lambda x: len([c for c in str(x)if c in string.punctuation]))
df['hashtag_count'] = df['hashtags'].apply(lambda x: len(str(x ).split()))
df['mention_count'] = df['mentions'].apply(lambda x: len(str(x ).split()))
df['link_count'] = df['links'].apply(lambda x: len(str(x ).split()))
df['caps_count'] = df['text_clean'].apply(lambda x: sum(1 for c in str(x)if c.isupper()))
df['caps_ratio'] = df['caps_count'] / df['text_len']
return df
train = create_stat(train)
 test = create_stat(test)
 print(train.shape, test.shape )<sort_values>
test_data = test_data.drop(['id'],axis=1 )
train.corr() ['target'].drop('target' ).sort_values()<define_variables>
X.isnull().all().unique()
stopwords = set(STOPWORDS)
<categorify>
y_train.isnull().any()
features = ['keyword', 'location_clean']
encoder = ce.TargetEncoder(cols=features)
 encoder.fit(train[features],train['target'])
 train = train.join(encoder.transform(train[features] ).add_suffix('_target'))
test = test.join(encoder.transform(test[features] ).add_suffix('_target'))<train_on_grid>
X_temp = X.values.reshape(X.shape[0], 28, 28 )
features_to_drop = ['id', 'keyword','location','text','location_clean','text_clean', 'hashtags', 'mentions','links']
X_train = train.drop(columns = features_to_drop + ['target'])
 X_test = test.drop(columns = features_to_drop)
 y_train = train.target
lr = LogisticRegression(solver = 'liblinear')
 lr.fit(X_train, y_train)
 cv5 = cross_val_score(lr, X_train, y_train, cv=5, scoring='f1')
 print(cv5)
 print(np.mean(cv5))<save_to_csv>
from keras.preprocessing.image import ImageDataGenerator
y_test = lr.predict(X_test)
 submit = sub_sample.copy()
submit.target = y_test
submit.to_csv('submit_lr.csv',index=False )<load_from_csv>
datagen = ImageDataGenerator(
 rotation_range= 8,
zoom_range = 0.12,
width_shift_range=0.1,
height_shift_range=0.1)
 datagen.fit(X )
submit_lr = pd.read_csv(".. /input/socialmediadisastertweetsdfecsv/submit_lr.csv" )<load_from_csv>
from keras.models import Sequential
from keras.layers import Conv2D, Dense, Dropout, BatchNormalization, Flatten, MaxPool2D
from keras.utils.np_utils import to_categorical
socialmedia_disaster_tweets-DFE = pd.read_csv(".. /input/disasters-on-social-media/socialmedia-disaster-tweets-DFE.csv" )<import_modules>
y_train = to_categorical(y_train,num_classes=10)
 checkpoint = ModelCheckpoint('BWeight.md5',monitor='val_loss',
save_best_only=True )
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename)) 
<load_from_csv>
model = Sequential()
model.add(Conv2D(32,kernel_size=3,activation='relu',input_shape=(28,28,1)))
 model.add(Conv2D(32,kernel_size=3,activation='relu'))
model.add(MaxPool2D())
 model.add(Conv2D(32,kernel_size=5,strides=2,padding='same',activation='relu'))
model.add(Dropout(0.4))
model.add(Conv2D(64,kernel_size=3,activation='relu'))
model.add(Conv2D(64,kernel_size=3,activation='relu'))
model.add(MaxPool2D())
 model.add(Conv2D(64,kernel_size=5,strides=2,padding='same',activation='relu'))
model.add(Dropout(0.4))
model.add(Flatten())
 model.add(Dense(128, activation='relu'))
model.add(Dropout(0.4))
model.add(Dense(10, activation='softmax'))
train_data = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 test_data = pd.read_csv('.. /input/nlp-getting-started/test.csv')
 sample_submission = pd.read_csv('.. /input/nlp-getting-started/sample_submission.csv')
 train_data<groupby>
model.compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"] )
count_data = train_data[['text','target']].groupby('target' ).count().reset_index()
count_data<count_missing_values>
X_train, X_val1, y_train, y_val1 = train_test_split(
 X, y_train, test_size=0.05, random_state=42 )
print(dataset.isnull().sum())
 print('-'*100)
 print(train_data.isnull().sum() )<feature_engineering>
size_batch = 64
dataset['len_letters'] = dataset['text'].apply(len)
 train_data['len_letters'] = train_data['text'].apply(len)
 dataset.head(5 )<data_type_conversions>
history = model.fit_generator(datagen.flow(X_train,y_train, batch_size=size_batch),
epochs = 60,
validation_data =(X_val1,y_val1),
verbose = 2,
steps_per_epoch = X_train.shape[0] // size_batch,
callbacks=[checkpoint] )
dataset['keyword'] = dataset['keyword'].fillna('not_known' )<filter>
model.load_weights('BWeight.md5' )
dataset['text'][dataset['keyword'] == 'blaze']<count_missing_values>
extra_validation = pd.read_csv('/kaggle/input/Kannada-MNIST/Dig-MNIST.csv' )
train_data['location'].isnull().sum()<count_missing_values>
y_extra_validate = extra_validation['label']
X_extra_validate = extra_validation.drop(['label'],axis=1 )
print('For disastrous, unknown locations : ',train_data['location'][train_data['target']==1].isnull().sum())
 print('For non disastrous, unknown locations : ',train_data['location'][train_data['target']==0].isnull().sum())
<categorify>
from sklearn.metrics import classification_report
dataset['location'].fillna('unknown', inplace = True)
 train_data['location'].fillna('unknown', inplace = True )<define_variables>
print(classification_report(y_extra_validate,model.predict_classes(X_extra_validate)) )
num_locations = Counter(dataset['location'])
 num_locations 
<categorify>
FINAL_PREDS = model.predict_classes(test_data )
dataset['location'] = le().fit_transform(dataset['location'])
 dataset<feature_engineering>
submission = pd.DataFrame({ 'id': Id,
'label': FINAL_PREDS })
 submission.to_csv(path_or_buf ="Kannada_MNIST_KERAS.csv", index=False )