Kaggle competition: Santander Value Prediction Challenge
<SOS> metric: RMSLE Kaggle data source: santander-value-prediction-challenge<load_pretrained>
warnings.filterwarnings('ignore')
learn_classifier = text_classifier_learner(data_clas, AWD_LSTM, drop_mult=0.5)
 learn_classifier.load_encoder('fine_tuned_enc')
 learn_classifier.freeze() 
<find_best_params>
import torch
import torch.nn as nn
import torch.optim as optim
from torch.autograd import Variable
learn_classifier.lr_find() 
<train_model>
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv' )
learn_classifier.fit_one_cycle(5, 2e-2, moms=(0.8,0.7)) 
<train_model>
y = np.log1p(train_df.target.values )
learn_classifier.freeze_to(-2)
 learn_classifier.fit_one_cycle(5, slice(1e-2/(2.6**4),1e-2), moms=(0.8,0.7)) 
<train_model>
train_df.drop(["ID", 'target'], axis=1, inplace = True)
 test_df.drop(["ID"], axis=1, inplace = True )
learn_classifier.freeze_to(-3)
 learn_classifier.fit_one_cycle(5, slice(5e-3/(2.6**4),5e-3), moms=(0.8,0.7))<find_best_params>
df = pd.concat([train_df, test_df] ).reset_index(drop=True)
 del train_df, test_df
gc.collect() ;
learn_classifier.show_results() 
<predict_on_test>
df.loc[:, :] = np.log1p(df.values )
preds, target = learn_classifier.get_preds(DatasetType.Test, ordered=True)
 labels = np.argmax(preds, axis =1)
<save_to_csv>
SEED = 717
np.random.seed = SEED
submission = pd.DataFrame({'PhraseId': test_id, 'Sentiment': labels})
 submission.to_csv('submission.csv', index=False)
 submission.head()<choose_model_class>
COMPONENTS = 20
methods = [
TruncatedSVD(n_components=COMPONENTS),
PCA(n_components=COMPONENTS),
GaussianRandomProjection(n_components=COMPONENTS, eps=0.1, random_state = SEED + 354),
]
embeddings = []
for method in methods:
name = method.__class__.__name__
embeddings.append(
 pd.DataFrame(method.fit_transform(df), columns=[f"{name}_{i}" for i in range(COMPONENTS)])
)
 print(f">> Ran {name}")
 X = pd.concat(embeddings, axis=1 ).reset_index(drop=True)
 del embeddings, method, methods, name
gc.collect() ;
lemmatizer = WordNetLemmatizer() 
<set_options>
%%time
dropzero = []
for row in np.arange(df.shape[0]):
dropzero.append(df.values[row, np.nonzero(df.values[row])])
 additional_df = pd.DataFrame(index=df.index)
 additional_df['mean'] = [x.mean() for x in dropzero]
additional_df['std'] = [x.std(ddof=1)for x in dropzero]
additional_df['var'] = [x.var(ddof=1)for x in dropzero]
additional_df['q_0_25'] = [np.percentile(x, q = 25)for x in dropzero]
additional_df['q_0_50'] = [np.percentile(x, q = 50)for x in dropzero]
additional_df['q_0_75'] = [np.percentile(x, q = 75)for x in dropzero]
set_random_seed(123)
 random.seed(123)
<load_from_csv>
additional_df.fillna(additional_df.mean(axis = 1), inplace=True )
train= pd.read_csv(".. /input/train.tsv", sep="\t")
 test = pd.read_csv(".. /input/test.tsv", sep="\t")
 train.head()<string_transform>
X = X.join(additional_df ).fillna(0)
 del additional_df
gc.collect() ;
def clean_sentences(df):
reviews = []
for sent in tqdm(df['Phrase']):
review_text = BeautifulSoup(sent ).get_text()
review_text = re.sub("[^a-zA-Z]"," ", review_text)
 words = word_tokenize(review_text.lower())
 lemma_words = [lemmatizer.lemmatize(i)for i in words]
reviews.append(lemma_words)
 return(reviews)
<string_transform>
USE_GPU = True
if USE_GPU and torch.cuda.is_available() :
print('using device: cuda')
 else:
print('using device: cpu' )
train_sentences = clean_sentences(train)
 test_sentences = clean_sentences(test)
 print(len(train_sentences))
print(len(test_sentences))<categorify>
torchm = nn.Sequential(
 nn.Linear(X.shape[1], 1024),
nn.Linear(1024, 512),
nn.Linear(512, 1))
optimizer = optim.Adam(torchm.parameters() , lr=0.001)
 criterion = nn.MSELoss()
target=train.Sentiment.values
y_target=to_categorical(target)
 num_classes=y_target.shape[1]<split>
if USE_GPU and torch.cuda.is_available() :
X = torch.from_numpy(X.values ).cuda()
y = torch.from_numpy(y ).view(-1,1 ).cuda()
dtype = torch.cuda.FloatTensor
torchm.cuda()
else:
X = torch.from_numpy(X.values)
 y = torch.from_numpy(y ).view(-1,1)
 dtype = torch.FloatTensor
X = Variable(X ).type(dtype)
 y = Variable(y ).type(dtype )
X_train,X_val,y_train,y_val=train_test_split(train_sentences,y_target,test_size=0.2,stratify=y_target )<count_unique_values>
torch.cuda.manual_seed_all(SEED)
 epoches = 126
for epoch in np.arange(epoches):
def closure() :
optimizer.zero_grad()
out = torchm(X[:y.shape[0]])
 loss = criterion(out, y)**.5
loss.backward()
return loss
loss = optimizer.step(closure)
 if epoch % 25 == 0:
print(f"RMSE for {epoch} epoch : {loss.data[0]}" )
unique_words = set()
len_max = 0
for sent in tqdm(X_train):
unique_words.update(sent)
 if(len_max<len(sent)) :
len_max = len(sent)
 print(len(list(unique_words)))
 print(len_max )<string_transform>
preds = torchm(X[y.shape[0]:] )
tokenizer = Tokenizer(num_words=len(list(unique_words)))
 tokenizer.fit_on_texts(list(X_train))
X_train = tokenizer.texts_to_sequences(X_train)
 X_val = tokenizer.texts_to_sequences(X_val)
 X_test = tokenizer.texts_to_sequences(test_sentences)
 X_train = sequence.pad_sequences(X_train, maxlen=len_max)
 X_val = sequence.pad_sequences(X_val, maxlen=len_max)
 X_test = sequence.pad_sequences(X_test, maxlen=len_max)
 print(X_train.shape,X_val.shape,X_test.shape )<choose_model_class>
if USE_GPU and torch.cuda.is_available() :
preds = preds.cpu().data.numpy()
else:
preds = preds.data.numpy()
<choose_model_class><EOS> 
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub.target = np.expm1(preds)
 sub.to_csv('torch.csv', index=False )