Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<load_from_csv>
train_data = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 test_data = pd.read_csv('.. /input/nlp-getting-started/test.csv')
 train_data.head()
trainingDartDf=pd.read_csv('.. /input/writefeaturetablefromsmotedartset/trainingFeatures1039.csv')
 trainingJimsDf=pd.read_csv('.. /input/aggregatecustomfeatures/trainingAggregate.csv')
 if 'Unnamed: 0' in trainingDartDf.columns:
trainingDartDf=trainingDartDf.drop('Unnamed: 0', axis=1)
 print(trainingDartDf.shape)
<define_variables>
def decontracted(phrase):
phrase = re.sub(r"won't", "will not", phrase)
 phrase = re.sub(r"can't", "can not", phrase)
 phrase = re.sub(r"n't", " not", phrase)
 phrase = re.sub(r"'re", " are", phrase)
 phrase = re.sub(r"'s", " is", phrase)
 phrase = re.sub(r"'d", " would", phrase)
 phrase = re.sub(r"'ll", " will", phrase)
 phrase = re.sub(r"'t", " not", phrase)
 phrase = re.sub(r"'ve", " have", phrase)
 phrase = re.sub(r"'m", " am", phrase)
 return phrase
columnsToAdd=['pb0maxstd', 'pb0medmed', 'pb0maxavg', 'pb0maxdsl',
'pb3maxstd', 'pb3medmed', 'pb3maxavg', 'pb3maxdsl', 'pb3transitory',
'pb0transitory', 'pb1maxstd', 'pb1medmed', 'pb1maxavg', 'pb1maxdsl',
'pb4maxstd', 'pb4medmed', 'pb4maxavg', 'pb4maxdsl', 'pb4transitory',
'pb1transitory', 'pb2maxstd', 'pb2medmed', 'pb2maxavg', 'pb2maxdsl',
'pb5maxstd', 'pb5medmed', 'pb5maxavg', 'pb5maxdsl', 'pb5transitory',
'pb2transitory', 'irmaxstd', 'vismaxstd', 'uvmaxstd', 'irmedmed',
'vismedmed', 'uvmedmed', 'irmaxavg', 'vismaxavg', 'uvmaxavg',
'irmaxdsl', 'vismaxdsl', 'uvmaxdsl', 'irtransitory', 'vistransitory',
'uvtransitory', 'irSpread', 'uvSpread', 'visSpread', 'irUvSpreadRatio',
'irMinusUvTransitory', 'irVisSpreadRatio', 'irMinusVisTransitory']
for column in columnsToAdd:
trainingDartDf.loc[:,column]=trainingJimsDf.loc[:,column]
traindf=trainingDartDf
del traindf['hostgal_specz']
del traindf['ra'], traindf['decl'], traindf['gal_l'], traindf['gal_b']
del traindf['ddf']
print(traindf.shape)
 traindf.head()<load_from_csv>
nlp = spacy.load('en')
 def preprocessing(text):
text = text.replace('
text = decontracted(text)
 text = re.sub('\S*@\S*\s?','',text)
 text = re.sub('http[s]?:(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F])) +','',text)
 token=[]
result=''
text = re.sub('[^A-z]', ' ',text.lower())
 text = nlp(text)
 for t in text:
if not t.is_stop and len(t)>2:
token.append(t.lemma_)
 result = ' '.join([i for i in token])
 return result.strip()
testDartDf=pd.read_csv('.. /input/writefeaturetablefromsmotedartset/feat_0.648970_2018-11-23-09-00.csv')
 testJimsDf=pd.read_csv('.. /input/fork-of-aggregatecustomfeaturestest/testAggregate.csv')
 if 'Unnamed: 0' in testDartDf.columns:
testDartDf=testDartDf.drop('Unnamed: 0', axis=1)
 print(testDartDf.shape)
 testDartDf.head()
for column in columnsToAdd:
testDartDf[column]=testJimsDf[column]
testdf=testDartDf
del testdf['hostgal_specz']
del testdf['ra'], testdf['decl'], testdf['gal_l'], testdf['gal_b']
del testdf['ddf']
testdf.shape<train_model>
data.text = data.text.apply(lambda x : preprocessing(x))
for cindex in full_train.columns:
full_train.loc[:,cindex]=np.float64(full_train.loc[:,cindex])
 eval_func = partial(lgbm_modeling_cross_validation,
full_train=full_train,
y=y,
classes=classes,
class_weights=class_weights,
nr_fold=5,
random_state=1)
 best_params.update({'n_estimators': 2000})
 clfs, score, oof_preds, importances = eval_func(best_params )<load_from_csv>
stop=set(stopwords.words('english'))
def create_corpus(df):
corpus=[]
for tweet in tqdm(df['text']):
words=[word.lower() for word in word_tokenize(tweet)if(( word.isalpha() ==1)&(word not in stop)) ]
corpus.append(words)
 return corpus
filename = 'subm_{:.6f}_{}.csv'.format(score,
dt.now().strftime('%Y-%m-%d-%H-%M'))
print('save to {}'.format(filename))
process_test(clfs,
testdf,
full_train,
train_mean=train_mean,
filename=filename,
chunks=40615)
 pdf = pd.read_csv('predictions.csv' )<save_to_csv>
corpus=create_corpus(data )
cols = list(pdf)
 cols.insert(0, cols.pop(cols.index('object_id')))
 pdf = pdf.loc[:, cols]
feats = ['class_6', 'class_15', 'class_16', 'class_42', 'class_52', 'class_53',
'class_62', 'class_64', 'class_65', 'class_67', 'class_88', 'class_90',
'class_92', 'class_95']
y = pd.DataFrame()
y['mymean'] = pdf[feats].mean(axis=1)
 y['mymedian'] = pdf[feats].median(axis=1)
 y['mymax'] = pdf[feats].max(axis=1)
 pdf['class_99'] = GenUnknown(y)
 pdf.to_csv(filename, index=False)
<define_variables>
MAX_LEN=40
tokenizer_obj=Tokenizer()
tokenizer_obj.fit_on_texts(corpus)
 sequences=tokenizer_obj.texts_to_sequences(corpus)
 tweet_pad=pad_sequences(sequences,maxlen=MAX_LEN,truncating='post',padding='post' )
COMPLETE_RUN = False<set_options>
word_index=tokenizer_obj.word_index
print('Number of unique words:',len(word_index))
np.random.seed(1001)
 %matplotlib inline
matplotlib.style.use('ggplot' )<load_from_csv>
num_words=len(word_index)+1
embedding_matrix=np.zeros(( num_words,200))
for word,i in tqdm(word_index.items()):
if i > num_words:
continue
emb_vec=embedding_dict.get(word)
 if emb_vec is not None:
embedding_matrix[i]=emb_vec
train = pd.read_csv(".. /input/freesound-audio-tagging/train.csv")
 test = pd.read_csv(".. /input/freesound-audio-tagging/sample_submission.csv" )<count_unique_values>
model=Sequential()
embedding=Embedding(num_words,200,embeddings_initializer=Constant(embedding_matrix),
input_length=MAX_LEN,trainable=False)
 model.add(embedding)
 model.add(SpatialDropout1D(0.2))
model.add(LSTM(64,dropout=0.2, recurrent_dropout=0.2))
model.add(Dense(1, activation='sigmoid'))
optimzer=Adam(learning_rate=0.001)
 model.compile(loss='binary_crossentropy',optimizer=optimzer,metrics=['accuracy'])
 model.summary()
print("Number of training examples=", train.shape[0], " Number of classes=", len(train.label.unique()))<count_values>
filepath="weights-improvement-{epoch:02d}-{val_accuracy:.2f}.hdf5"
checkpoint = ModelCheckpoint(filepath, monitor='val_accuracy', verbose=1, save_best_only=True, mode='max')
 es = EarlyStopping(monitor='val_accuracy', mode='max', verbose=1, patience=30)
 callbacks_list = [checkpoint,es]
print('Minimum samples per category = ', min(train.label.value_counts()))
print('Maximum samples per category = ', max(train.label.value_counts()))<load_pretrained>
tweet =data.iloc[:7613,:]
test = data.iloc[7613:,:]
fname = '.. /input/freesound-audio-tagging/audio_train/' + '00044347.wav'
ipd.Audio(fname )<normalization>
train=tweet_pad[:tweet.shape[0]]
test=tweet_pad[tweet.shape[0]:]
wav = wave.open(fname)
 print("Sampling(frame)rate = ", wav.getframerate())
 print("Total samples(frames)= ", wav.getnframes())
 print("Duration = ", wav.getnframes() /wav.getframerate() )<train_model>
X_train,X_test,y_train,y_test=train_test_split(train,tweet['target'].values,test_size=0.2)
 print('Shape of train',X_train.shape)
 print("Shape of Validation ",X_test.shape )
rate, data = wavfile.read(fname)
 print("Sampling(frame)rate = ", rate)
 print("Total samples(frames)= ", data.shape)
 print(data )<import_modules>
history=model.fit(X_train,y_train,batch_size=100,epochs=20,callbacks=callbacks_list,validation_data=(X_test,y_test),verbose=1 )
ModelCheckpoint, TensorBoard, ReduceLROnPlateau)
 GlobalMaxPool1D, Input, MaxPool1D, concatenate)
<set_options>
sample_sub=pd.read_csv('.. /input/nlp-getting-started/sample_submission.csv' )
<define_variables><EOS> 
y_pre = model.predict(test)
 y_pre=np.round(y_pre ).astype(int ).reshape(3263)
 sub=pd.DataFrame({'id':sample_sub['id'].values.tolist() ,'target':y_pre})
 sub.to_csv('submission.csv',index=False )