Kaggle competition: Two Sigma Connect: Rental Listing Inquiries
<SOS> metric: MulticlassLoss Kaggle data source: two-sigma-connect:-rental-listing-inquiries<categorify>

 def load_json_from_path(path):
try:
with open(path, "r")as f:
return json.load(f)
 except OSError as error:
if error.errno != errno.ENOENT:
raise error
return None
def json_to_pandas_dataframe(dictionary):
columns = list(dictionary.keys())
 rows = sorted(list(set(itertools.chain.from_iterable([
list(dictionary[k].keys())
 for k in columns
]))), key=lambda x: int(x))
data = list(map(list, zip(*([rows] + [
[
dictionary[column][r] if r in dictionary[column] else None
for r in rows
]
for column in columns
]))))
df = pd.DataFrame(data, columns=['id'] + columns)
 df.set_index('id')
 return df
def load_training_test_data(training_data_path, test_data_path):
return(
 json_to_pandas_dataframe(load_json_from_path(training_data_path)) ,
json_to_pandas_dataframe(load_json_from_path(test_data_path)) 
)

def string_to_category_name(string):
return string.lower().replace(" ", "_")
 def categories_from_column(data_frame, column):
return list(set(list(itertools.chain.from_iterable(
 data_frame[column].tolist() 
))))
def normalize_whitespace(string):
return re.sub(r"\s+", " ", string)
 def normalize_category(category):
return normalize_whitespace(re.sub(r"[\*\-\!\&]", " ", category.lower())).strip()
def normalize_categories(categories):
return [
normalize_category(c)for c in categories
]
def sliding_window(sequence, n):

 it = iter(sequence)
 window = deque(( next(it, None)for _ in range(n)) , maxlen=n)
 yield list(window)
 for element in it:
window.append(element)
 yield list(window)
 def create_ngrams(content, n):
for ngram in sliding_window(content.split() , n):
yield " ".join(ngram)
 def create_ngrams_up_to_n(content, n):
for i in range(n):
yield from create_ngrams(content, i)
 def count_ngrams_up_to_n(content, n):
return Counter(list(create_ngrams_up_to_n(content, n)))
 def remove_small_or_stopwords_from_ranking(ranking, nlp, min_len):
for word, rank in ranking:
if nlp.vocab[word].is_stop or len(word)< min_len:
continue
yield word, rank
def column_list_to_category_flags(data_frame, column, grams):
categories = [
"{}_{}".format(column, string_to_category_name(n))
for n in grams
]
row_cleaned_categories = [
normalize_category(" ".join(r))
for r in data_frame[column].tolist()
]
category_flags = pd.DataFrame.from_records([
[1 if gram in r else 0 for gram in grams]
for r in row_cleaned_categories
], columns=categories)
 return pd.concat(( data_frame, category_flags), axis=1)
 def remap_column(data_frame, column, new_column, mapping):
data_frame[new_column] = data_frame[column].transform(mapping)
 return data_frame
def remap_date_column_to_days_before(data_frame,
column,
new_column,
reference_date):
data_frame[new_column] = data_frame[column].transform(
 lambda x:(reference_date - datetime.datetime.strptime(x, "%Y-%m-%d %H:%M:%S")).days 
)
 return data_frame
def map_categorical_column_to_category_ids(train_data_frame,
test_data_frame,
column,
new_column,
min_freq=1):
category_counts = Counter(train_data_frame[column])+ Counter(test_data_frame[column])
 category_to_unknown_mapping = {
category: category if count >= min_freq else "Unknown"
for category, count in category_counts.items()
}
category_to_id_map = {
category: i
for i, category in enumerate(sorted([
category_to_unknown_mapping[c] for c in 
(set(train_data_frame[column])| set(test_data_frame[column]))
]))
}
id_to_category_map = {
i: category
for category, i in category_to_id_map.items()
}
return(
 category_to_unknown_mapping,
category_to_id_map,
id_to_category_map,
remap_column(train_data_frame,
column,
new_column,
lambda x: category_to_id_map[category_to_unknown_mapping[x]]),
remap_column(test_data_frame,
column,
new_column,
lambda x: category_to_id_map[category_to_unknown_mapping[x]])
)
 def remap_columns_with_transform(train_data_frame,
test_data_frame,
column,
new_column,
transform):

 return(
 remap_column(train_data_frame,
column,
new_column,
transform),
remap_column(test_data_frame,
column,
new_column,
transform)
)
 def normalize_description(description):

 description = description.lower()
description = re.sub(r"<[^<]+?(>|$)", " ", description)
 description = re.sub(r"[0-9\-]+", " ", description)
 description = re.sub(r"[a-z0-9]@[a-z0-9]\.[a-z]", " ", description)
 description = re.sub(r"[\!]+", "! ", description)
 description = re.sub(r"[\-\:]", " ", description)
 description = re.sub("\*", " ", description)
 return re.sub(r"\s+", " ", description)
 def add_epsilon(array):
return np.array([a + 10e-10 if a == 0 else a for a in array])
 def numerical_feature_engineering_on_dataframe(dataframe,
numerical_columns):

 for lhs_column, rhs_column in itertools.combinations(numerical_columns, 2):
dataframe['{}_add_{}'.format(lhs_column, rhs_column)] = dataframe[lhs_column] + dataframe[rhs_column]
dataframe['{}_sub_{}'.format(lhs_column, rhs_column)] = dataframe[lhs_column] - dataframe[rhs_column]
dataframe['{}_mul_{}'.format(lhs_column, rhs_column)] = dataframe[lhs_column] * dataframe[rhs_column]
dataframe['{}_div_{}'.format(lhs_column, rhs_column)] = dataframe[lhs_column] / add_epsilon(dataframe[rhs_column])
 return dataframe
def numerical_feature_engineering(train_data_frame,
test_data_frame,
numerical_columns):

 return(
 numerical_feature_engineering_on_dataframe(train_data_frame,
numerical_columns),
numerical_feature_engineering_on_dataframe(test_data_frame,
numerical_columns), 
)
 def normalize_eastwest(eastwest):
eastwest = eastwest.lower().strip()
if not eastwest:
return ""
if eastwest[0] == "e":
return "e"
elif eastwest[0] == "w":
return "w"
else:
return ""
STREET_MAPPING = {
"st": "street",
"ave": "avenue",
"pl": "place",
"rd": "road"
}
def normalize_name(name):
m = re.match(r"(?P<address>[\w\s]+ )(?P<st>st|street|ave|avenue|place|pl|road|rd ).*",
name.lower().strip())
 if not m:
return name.lower().strip()
return "{address} {street}".format(
 address=m.groupdict() ["address"].strip() ,
street=STREET_MAPPING.get(m.groupdict() ["st"], m.groupdict() ["st"])
)
 def normalize_address(address_dict):
return "{eastwest} {name}".format(
 eastwest=normalize_eastwest(address_dict["eastwest"] or ""),
name=normalize_name(address_dict["name"] or "")
)
 def parse_address_components_from_address(address):
m = re.match(r"(?P<number>[0-9]*\s+)?\s*(?P<eastwest>East|West|E\s|W\s)?\s*(?P<name>[A-Za-z0-9\.\-\s]*)",
normalize_whitespace(address),
flags=re.IGNORECASE)
 return {
"normalized": normalize_address(m.groupdict())if m is not None else address
}
def parse_address_components_for_column(dataframe, column):
return pd.concat(( dataframe, pd.DataFrame.from_records([
{
"{}_{}".format(column, key): value for key, value in
parse_address_components_from_address(cell ).items()
}
for cell in dataframe[column]
])) , axis=1)
 def parse_address_components(train_data_frame,
test_data_frame,
columns):
return(
 functools.reduce(lambda df, c: parse_address_components_for_column(df,
c),
columns,
train_data_frame),
functools.reduce(lambda df, c: parse_address_components_for_column(df,
c),
columns,
test_data_frame)
)
 def count_json_column(dataframe, column):
return pd.DataFrame([
len(c)for c in dataframe[column]
])
 def count_json(train_data_frame,
test_data_frame,
column):
train_data_frame["{}_count".format(column)] = count_json_column(train_data_frame,
column)
 test_data_frame["{}_count".format(column)] = count_json_column(test_data_frame,
column)
 return train_data_frame, test_data_frame

 def maybe_cuda(tensor):

 if torch.cuda.is_available() :
return tensor.cuda()
return tensor.cpu()
class Doc2Vec(nn.Module):

 def __init__(self,
embedding_size,
hidden_layer_size,
vocab_size,
output_size,
batch_size):
super().__init__()
self.hidden_dim = hidden_layer_size
self.embedding = nn.Embedding(vocab_size, embedding_size)
 self.batch_size = batch_size
self.embedding_size = embedding_size
self.hidden =(maybe_cuda(torch.randn(2, batch_size, self.hidden_dim)) ,
maybe_cuda(torch.randn(2, batch_size, self.hidden_dim)))
 self.lstm = nn.LSTM(embedding_size,
self.hidden_dim,
num_layers=1,
bidirectional=True)
 self.linear = nn.Linear(self.hidden_dim * 2, output_size)
 def sentence_embedding(self, sentence):
self.hidden =(maybe_cuda(torch.randn(2, self.batch_size, self.hidden_dim)) ,
maybe_cuda(torch.randn(2, self.batch_size, self.hidden_dim)))
 embeddings = self.embedding(sentence)
 out, self.hidden = self.lstm(embeddings.view(-1, self.batch_size, self.embedding_size),
self.hidden)
 added = self.hidden[0] + self.hidden[1]
return added / torch.norm(added)
 def forward(self, sentence):
embedding = self.sentence_embedding(sentence)
 lin = self.linear(embedding.view(-1, self.hidden_dim * 2))
return F.softmax(lin, dim=1)
 def train_model(model, optimizer, epochs, sentence_tensors, label_tensors):
for epoch in range(epochs):
total_loss = 0
loss_criterion = nn.CrossEntropyLoss()
shuffled_sentence_tensors, shuffled_label_tensors = shuffle(
 sentence_tensors, label_tensors 
)
 progressable_tensors = tqdm(
 zip(shuffled_sentence_tensors, shuffled_label_tensors),
total=len(shuffled_label_tensors),
desc="Processing sentence vectors" 
)
 for sentence_tensor, label_tensor in progressable_tensors:
optimizer.zero_grad()
preds = model(sentence_tensor)
 loss = loss_criterion(preds,
label_tensor)
 loss.backward()
optimizer.step()
total_loss += loss.item()
progressable_tensors.set_postfix(loss=loss.item())
 print('Epoch', epoch, 'total loss', total_loss)
 with torch.no_grad() :
print("Model Accuracy:",
compute_model_accuracy(model,
shuffled_sentence_tensors[0],
shuffled_label_tensors[0]))
return model
def characters_to_one_hot_lookups(all_characters):
set_characters = set(all_characters)
 character_to_one_hot = {
c: i for i, c in enumerate(sorted(set_characters))
}
one_hot_to_character = {
c: i for i, c in enumerate(sorted(set_characters))
}
return character_to_one_hot, one_hot_to_character
def character_sequence_to_matrix(sentence, character_to_one_hot):
return np.array([character_to_one_hot[c] for c in sentence])
 def compute_model_accuracy(model, sentence_tensors, label_tensors):

 predictions = np.argmax(model(maybe_cuda(sentence_tensors)).detach().cpu().numpy() , axis=1 ).flatten()
labels = label_tensors.detach().cpu().numpy().flatten()
return len([p for p in(predictions == labels)if p == True])/ len(predictions)
 def pad_sentence(sentence, padding):
truncated = sentence[:padding]
return truncated +(" " *(padding - len(truncated)))
 def to_batches(sentences, batch_size, pad_value):
for i in range(math.ceil(len(sentences)/ batch_size)) :
batch = [
sentences[i * batch_size + j]
for j in range(min(( batch_size, len(sentences[i * batch_size:]))))
]
padding = [
pad_value
for k in range(max(0, batch_size - len(sentences[i * batch_size:])))
 ]
yield torch.stack(batch + padding, dim=0)
 def documents_to_vectors_model(train_documents,
test_documents,
labels,
epochs,
parameters,
learning_rate,
load=None,
save=None,
sentence_length=1000,
batch_size=200):

 character_to_one_hot, one_hot_to_character = characters_to_one_hot_lookups(
 "".join(train_documents)+ "".join(test_documents)
)
 train_sentence_tensors = list(to_batches([
maybe_cuda(torch.tensor(character_sequence_to_matrix(pad_sentence(sentence, sentence_length),
character_to_one_hot), dtype=torch.long))
for sentence in train_documents
], batch_size, maybe_cuda(torch.tensor([character_to_one_hot[" "] for i in range(sentence_length)]))))
label_tensors = list(to_batches([maybe_cuda(torch.tensor(i)) for i in labels],
batch_size,
maybe_cuda(torch.tensor(0))))
model = maybe_cuda(Doc2Vec(parameters,
parameters * 2,
len(character_to_one_hot.keys()), max(labels)+ 1,
batch_size))
if not load:
optimizer = optim.Adam(model.parameters() , lr=learning_rate)
 try:
train_model(model,
optimizer,
epochs,
train_sentence_tensors,
label_tensors)
 except KeyboardInterrupt:
print("Interrupted, saving current state now")
 else:
model.load_state_dict(torch.load(load))
if save:
torch.save(model.state_dict() , save)
 return character_to_one_hot, one_hot_to_character, model
def generate_document_vector_embeddings_from_model(model,
character_to_one_hot,
sentences,
sentence_length,
batch_size):
sentence_tensors = list(to_batches([
maybe_cuda(torch.tensor(character_sequence_to_matrix(pad_sentence(sentence, sentence_length),
character_to_one_hot), dtype=torch.long))
for sentence in sentences
], batch_size, maybe_cuda(torch.tensor([character_to_one_hot[" "] for i in range(sentence_length)]))))
with torch.no_grad() :
return np.row_stack([
model.sentence_embedding(sentence_tensor ).detach().cpu().numpy()
for sentence_tensor in sentence_tensors
] ).reshape(-1, model.hidden_dim * 2)[:len(sentences)]
def column_to_doc_vectors(train_data_frame,
test_data_frame,
description_column,
target_column,
document_vector_column,
epochs=100,
parameters=40,
learning_rate=0.01,
load=None,
save=None,
sentence_length=1000,
batch_size=200):

 train_descriptions = list(train_data_frame[description_column])
 test_descriptions = list(test_data_frame[description_column])
 labels = list(train_data_frame[target_column])
 character_to_one_hot, one_hot_to_character, model = documents_to_vectors_model(
 train_descriptions,
test_descriptions,
labels,
epochs,
parameters,
learning_rate,
load=load,
save=save,
sentence_length=sentence_length,
batch_size=batch_size 
)
 train_description_vectors = pd.DataFrame(
 generate_document_vector_embeddings_from_model(
 model,
character_to_one_hot,
train_descriptions,
sentence_length,
batch_size 
)
)
 test_description_vectors = pd.DataFrame(
 generate_document_vector_embeddings_from_model(
 model,
character_to_one_hot,
test_descriptions,
sentence_length,
batch_size 
)
)
 return(
 pd.concat(( train_data_frame, train_description_vectors), axis=1),
pd.concat(( test_data_frame, test_description_vectors), axis=1)
)

def sklearn_pipeline_steps(categorical_columns, verbose=False):
return(
('one_hot',
OneHotEncoder(cols=categorical_columns, verbose=verbose)) , 
('scaling', StandardScaler())
)
 def basic_logistic_regression_pipeline(categorical_columns,
verbose=False):
return Pipeline((
*sklearn_pipeline_steps(categorical_columns, verbose=verbose), 
('xgb', xgb.XGBClassifier(
 n_estimators=1000,
seed=42,
objective='multi:softprob',
subsample=0.8,
colsample_bytree=0.8, 
)) 
))
def calculate_statistics(statistics, test_labels, predictions):
return {
k: s(test_labels, predictions)
 for k, s in statistics.items()
}
def format_statistics(calculated_statistics):
return ", ".join([
"{0}: {1:.2f}".format(k, s)
 for k, s in calculated_statistics.items()
])
 def prediction_accuracy(labels, predictions):
return(
 len([a for a, b in zip(labels, predictions)if a == b])/ len(predictions)
)
 def fit_one_split(model, features, labels, statistics, train_index, test_index):
train_data, train_labels = features.iloc[train_index], labels[train_index]
test_data, test_labels = features.iloc[test_index], labels[test_index]
model.fit(train_data, train_labels)
 predictions = model.predict(test_data)
 return(
 test_labels,
predictions,
calculate_statistics(statistics, test_labels, predictions)
)
 def test_model_with_k_fold_cross_validation(model,
features,
labels,
statistics,
n_splits=5,
random_state=42):
kf = KFold(n_splits=n_splits, shuffle=True, random_state=random_state)
 test_labels, predictions = [], []
for i,(train_index, test_index)in enumerate(kf.split(features, labels)) :
fold_test_labels, fold_predictions, calculated_statistics = fit_one_split(
 model,
features,
labels,
statistics,
train_index,
test_index 
)
 print('Fold', i, format_statistics(calculated_statistics))
test_labels.extend(fold_test_labels)
 predictions.extend(fold_predictions)
 return(
 calculate_statistics(statistics, test_labels, predictions),
test_labels,
predictions,
model 
)
 def get_prediction_probabilities_with_columns(model,
test_dataframe,
keep_columns):
return pd.concat(( test_dataframe[keep_columns],
pd.DataFrame(model.predict_proba(test_dataframe.drop(keep_columns, axis=1)))) ,
axis=1)
y_train = np_utils.to_categorical(y_train,10)
 X = X.reshape(-1,28,28,1)
 y = np_utils.to_categorical(y,10 )<define_variables>
nlp = spacy.load("en" )
seed = 43
np.random.seed(seed )<import_modules>
torch.cuda.is_available()
from keras.models import Sequential
from keras.layers.core import Lambda , Dense, Flatten, Dropout
from keras.callbacks import EarlyStopping
from keras.layers import BatchNormalization, Convolution2D , MaxPooling2D<choose_model_class>
( TRAIN_DATAFRAME, TEST_DATAFRAME)= \
load_training_test_data(os.path.join('.. ', 'input', 'train.json'),
os.path.join('.. ', 'input', 'test.json'))
def get_bn_model() :
model = Sequential([
Lambda(standardize, input_shape=(28,28,1)) ,
Convolution2D(32,(3,3), activation='relu'),
BatchNormalization(axis=1),
Convolution2D(32,(3,3), activation='relu'),
MaxPooling2D() ,
BatchNormalization(axis=1),
Convolution2D(64,(3,3), activation='relu'),
BatchNormalization(axis=1),
Convolution2D(64,(3,3), activation='relu'),
MaxPooling2D() ,
Flatten() ,
BatchNormalization() ,
Dense(512, activation='relu'),
BatchNormalization() ,
Dense(10, activation='softmax')
 ])
 model.compile(Adam() , loss='categorical_crossentropy', metrics=['accuracy'])
 return model<train_model>
normalized_categories = sorted(normalize_categories(categories_from_column(TRAIN_DATAFRAME, 'features')))
 normalized_categories[:50]
model= get_bn_model()
gen =ImageDataGenerator(rotation_range=8, width_shift_range=0.08, shear_range=0.3,
height_shift_range=0.08, zoom_range=0.08)
 batches = gen.flow(X, y, batch_size=64)
 history=model.fit_generator(generator=batches, steps_per_epoch=batches.n, epochs=1)
<save_to_csv>
most_common_ngrams = sorted(count_ngrams_up_to_n(" ".join(normalized_categories), 3 ).most_common() ,
key=lambda x:(-x[1], x[0]))
most_common_ngrams[:50]
X_test.shape
results = model.predict(X_test)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label")
 submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("Aniket_Digit_pred2.csv",index=False )<set_options>
most_common_ngrams = sorted(list(remove_small_or_stopwords_from_ranking(most_common_ngrams, nlp, 3)) ,
key=lambda x:(-x[1], x[0]))
most_common_ngrams[:50]
warnings.filterwarnings("ignore")
 warnings.filterwarnings("ignore", category=DeprecationWarning )<load_from_csv>
TRAIN_DATAFRAME = column_list_to_category_flags(TRAIN_DATAFRAME, 'features', list(map(operator.itemgetter(0), most_common_ngrams[:100])))
 TEST_DATAFRAME = column_list_to_category_flags(TEST_DATAFRAME, 'features', list(map(operator.itemgetter(0), most_common_ngrams[:100])) )
img_train = '.. /input/cassava-leaf-disease-classification/train_images'
img_test = '.. /input/cassava-leaf-disease-classification/test_images'
base_weight = '.. /input/cassava-baseline/'
train_df = pd.read_csv('.. /input/cassava-train-folds/train_folds.csv')
 BATCH_SIZE = 64<categorify>
TRAIN_DATAFRAME = remap_date_column_to_days_before(TRAIN_DATAFRAME, "created", "created_days_ago", datetime.datetime(2017, 1, 1))
TEST_DATAFRAME = remap_date_column_to_days_before(TEST_DATAFRAME, "created", "created_days_ago", datetime.datetime(2017, 1, 1))
class TestDataset(Dataset):
def __init__(self,df,im_path,transforms=None):
self.df = df
self.im_path = im_path
self.transforms = transforms
def __getitem__(self,idx):
img_path = self.df.iloc[idx]['image_id']
img = Image.open(self.im_path+"/"+img_path)
 if self.transforms:
img = self.transforms(**{"image": np.array(img)})["image"]
return img
def __len__(self):
return self.df.shape[0]<choose_model_class>
TRAIN_DATAFRAME = remap_column(TRAIN_DATAFRAME, "interest_level", "label_interest_level", lambda x: {
"high": 0,
"medium": 1,
"low": 2
}[x])
 TEST_DATAFRAME["label_interest_level"] = 0
class Net(nn.Module):
def __init__(self,model_name='efficientnet-b3',pool_type=F.adaptive_avg_pool2d):
super().__init__()
self.pool_type = pool_type
self.backbone = EfficientNet.from_name(model_name)
 in_features = getattr(self.backbone,'_fc' ).in_features
self.classifier = nn.Linear(in_features,5)
 def forward(self,x):
features = self.pool_type(self.backbone.extract_features(x),1)
 features = features.view(x.size(0),-1)
 return self.classifier(features)
<define_search_space>
( BUILDING_ID_UNKNOWN_REMAPPING,
BUILDING_ID_TO_BUILDING_CATEGORY,
BUILDING_CATEGORY_TO_BUILDING_ID,
TRAIN_DATAFRAME,
TEST_DATAFRAME)= map_categorical_column_to_category_ids(
 TRAIN_DATAFRAME,
TEST_DATAFRAME,
'building_id',
'building_id_category',
min_freq=40
)
imagenet_stats = {'mean':[0.485, 0.456, 0.406], 'std':[0.229, 0.224, 0.225]}
test_tfms = A.Compose([
A.Resize(384,384,always_apply=1,p=1),
ToTensor(normalize=imagenet_stats)
 ] )<create_dataframe>
( MANAGER_ID_UNKNOWN_REMAPPING,
MANAGER_ID_TO_MANAGER_CATEGORY,
MANAGER_CATEGORY_TO_MANAGER_ID,
TRAIN_DATAFRAME,
TEST_DATAFRAME)= map_categorical_column_to_category_ids(
 TRAIN_DATAFRAME,
TEST_DATAFRAME,
'manager_id',
'manager_id_category',
min_freq=40
)
val_preds = []
val_targets = []
for fold in range(5):
df_t = train_df[train_df.kfold==fold].reset_index(drop=True)
 dataset_val = TestDataset(df_t,img_train,test_tfms)
 dataloader_val = DataLoader(dataset_val, batch_size=BATCH_SIZE, num_workers=4, shuffle=False)
 temp = pred(base_weight+f"fold{fold}.pth",dataloader_val)
 temp = torch.argmax(temp, dim=1)
 val_preds.extend(temp)
 val_targets.extend(df_t.label.values )<compute_test_metric>
TRAIN_DATAFRAME, TEST_DATAFRAME = parse_address_components(
 TRAIN_DATAFRAME,
TEST_DATAFRAME,
[
"display_address",
"street_address"
]
)
print(f"accuracy : {accuracy_score(val_targets,val_preds)}" )<load_from_csv>
( DISP_ADDR_ID_UNKNOWN_REMAPPING,
DISP_ADDR_TO_DISP_ADDR_CATEGORY,
DISP_ADDR_CATEGORY_TO_DISP_ADDR_ID,
TRAIN_DATAFRAME,
TEST_DATAFRAME)= map_categorical_column_to_category_ids(
 TRAIN_DATAFRAME,
TEST_DATAFRAME,
'display_address_normalized',
'display_address_category',
min_freq=10
)
submission_df = pd.read_csv('.. /input/cassava-leaf-disease-classification/sample_submission.csv')
 submission_df.iloc[:, 1] = 0
submission_df.head()<prepare_output>
TRAIN_DATAFRAME, TEST_DATAFRAME = count_json(
 TRAIN_DATAFRAME,
TEST_DATAFRAME,
"photos"
)
if submission_df.shape[0] == 1:
submission_df = pd.DataFrame([{'image_id': '2216849948.jpg', 'label': 0},{'image_id': '2216849948.jpg', 'label': 0}])
 submission_df.reset_index(drop=True, inplace=True)
 submission_df.head()<create_dataframe>
NUMERICAL_COLUMNS = [
'bathrooms',
'bedrooms',
'price',
'latitude',
'longitude',
'photos_count'
]
TRAIN_DATAFRAME, TEST_DATAFRAME = numerical_feature_engineering(
 TRAIN_DATAFRAME,
TEST_DATAFRAME,
NUMERICAL_COLUMNS
)
submissions=None
for fold in range(5):
dataset_test = TestDataset(submission_df,img_test,test_tfms)
 dataloader_test = DataLoader(dataset_test, batch_size=BATCH_SIZE, num_workers=4, shuffle=False)
 test_preds = pred(base_weight+f"fold{fold}.pth",dataloader_test)
 if submissions is None:
submissions = test_preds /5
else:
submissions+= test_preds /5<save_to_csv>
TRAIN_DATAFRAME, TEST_DATAFRAME = remap_columns_with_transform(
 TRAIN_DATAFRAME,
TEST_DATAFRAME,
'description',
'clean_description',
normalize_description
)
submission_df['label'] = torch.argmax(submissions, dim=1)
 submission_df.to_csv('submission.csv', index=False)
 submission_df 
<prepare_x_and_y>
DROP_COLUMNS = [
'id',
'created',
'building_id',
'clean_description',
'description',
'features',
'display_address',
'display_address_normalized',
'manager_id',
'photos',
'street_address',
'street_address_normalized',
'interest_level',
]
TRAIN_DATAFRAME = TRAIN_DATAFRAME.drop(DROP_COLUMNS, axis=1)
 TEST_DATAFRAME = TEST_DATAFRAME.drop(DROP_COLUMNS[:-1], axis=1 )
num_classes = 10
train_fname = '/kaggle/input/digit-recognizer/train.csv'
train_db = pd.read_csv(train_fname)
 label = train_db['label'].to_numpy()
label = to_categorical(label, num_classes=num_classes)
 data = train_db.drop('label', axis=1 ).to_numpy()
data = data.reshape(( data.shape[0], 28, 28, 1))
data = data.astype('float32')/ 255
x_train = data [:32000]
y_train = label[:32000]
x_valid = data [32000:]
y_valid = label[32000:]
del train_db<train_model>
[(i, x)for i, x in enumerate(np.all(np.isfinite(TRAIN_DATAFRAME.drop(['listing_id', 'label_interest_level'], axis=1)) , axis=0)) if not x]
train_datagen = ImageDataGenerator(rotation_range=10,
width_shift_range=0.1,
height_shift_range=0.1,
shear_range=0.1,
zoom_range=0.1)
 batch_size = 16
train_generator = train_datagen.flow(x_train, y_train, batch_size=batch_size )<choose_model_class>
TRAIN_DATAFRAME.drop(['listing_id', 'label_interest_level'], axis=1 ).columns[119]
model = Sequential([
Conv2D(32,(3, 3), activation='relu', input_shape=(28, 28, 1)) ,
Conv2D(32,(3, 3), activation='relu'),
SeparableConv2D(32,(5, 5), activation='relu', padding='same'),
BatchNormalization() ,
MaxPooling2D(( 2,2)) ,
SpatialDropout2D(0.35),
Conv2D(64,(3, 3), activation='relu'),
Conv2D(64,(3, 3), activation='relu'),
SeparableConv2D(64,(5, 5), activation='relu', padding='same'),
BatchNormalization() ,
MaxPooling2D(( 2,2)) ,
SpatialDropout2D(0.35),
Flatten() ,
Dense(256, activation='relu', kernel_regularizer=l2(1e-3)) ,
Dense(10, activation='softmax')
 ])
 model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['acc'])
 model.summary()<train_model>
np.argwhere(~np.isfinite(TEST_DATAFRAME.drop(['listing_id', 'label_interest_level'], axis=1 ).as_matrix()))
learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001)
 epochs = 30
performance = model.fit(train_generator,
steps_per_epoch=x_train.shape[0] // batch_size,
epochs=epochs,
validation_data=(x_valid, y_valid),
callbacks=[learning_rate_reduction] )<save_to_csv>
CATEGORICAL_COLUMNS =('building_id_category', 'manager_id_category' )
test_fname = '/kaggle/input/digit-recognizer/test.csv'
test_db = pd.read_csv(test_fname)
 x_test = test_db.to_numpy()
x_test = x_test.reshape(( x_test.shape[0], 28, 28, 1))
x_test = x_test.astype('float32')/ 255
predictions = np.argmax(model.predict(x_test), axis=1)
 imageIds = pd.Series(range(1,28001), name='ImageId')
 results = pd.Series(predictions , name='Label')
 submission = pd.concat([imageIds, results], axis=1)
 submission.to_csv("submission.csv", index=False, header=True)
<set_options>
( LINEAR_MODEL_STATISTICS,
LINEAR_MODEL_LABELS,
LINEAR_MODEL_PREDICTIONS,
LINEAR_MODEL)= test_model_with_k_fold_cross_validation(
 basic_logistic_regression_pipeline(CATEGORICAL_COLUMNS),
TRAIN_DATAFRAME.drop(['listing_id', 'label_interest_level'], axis=1 ).astype(float),
TRAIN_DATAFRAME['label_interest_level'],
{
"mse_loss": mean_squared_error,
"accuracy": prediction_accuracy
},
n_splits=2 
)
 print('Linear Model', format_statistics(LINEAR_MODEL_STATISTICS))
%matplotlib inline
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))<load_from_csv>
pd.set_option("display.max_columns",200)
 pd.set_option("display.max_rows",500)
 TRAIN_DATAFRAME[TRAIN_DATAFRAME.isnull().any(axis=1)]
df_train = pd.read_csv("/kaggle/input/digit-recognizer/train.csv")
 df_test = pd.read_csv("/kaggle/input/digit-recognizer/test.csv")
 df_submission = pd.read_csv("/kaggle/input/digit-recognizer/sample_submission.csv" )<count_missing_values>
table = get_prediction_probabilities_with_columns(LINEAR_MODEL,
TEST_DATAFRAME.drop('label_interest_level', axis=1),
['listing_id'])
 table.columns = ['listing_id', 'high', 'medium', 'low']
<data_type_conversions><EOS> 
table.to_csv('submission.csv', columns=['listing_id', 'high', 'medium', 'low'], index=False )