Kaggle competition: Histopathologic Cancer Detection
def find_bad_sitezero(X):

 return X[(X.timestamp < 3378)&(X.site_id == 0)&(X.meter == 0)].index<filter>
labels_path='.. /input/train_labels.csv'
labels = pd.read_csv(labels_path)
 labels.head()
def find_bad_building1099(X, y):

 return X[(X.building_id == 1099)&(X.meter == 2)&(y > 3e4)].index<compute_test_metric>
path=Path('.. /input/' )
def find_bad_rows(X, y):
return find_bad_zeros(X, y ).union(find_bad_sitezero(X)).union(find_bad_building1099(X, y))<data_type_conversions>
tfms=get_transforms(flip_vert=True, max_warp=0, max_rotate=0, max_lighting=0.05 )
pd.set_option("max_columns", 500)
 def input_file(file):
path = f".. /input/ashrae-energy-prediction/{file}"
if not os.path.exists(path): return path + ".gz"
return path
def compress_dataframe(df):
result = df.copy()
for col in result.columns:
col_data = result[col]
dn = col_data.dtype.name
if dn == "object":
result[col] = pd.to_numeric(col_data.astype("category" ).cat.codes, downcast="integer")
 elif dn == "bool":
result[col] = col_data.astype("int8")
 elif dn.startswith("int")or(col_data.round() == col_data ).all() :
result[col] = pd.to_numeric(col_data, downcast="integer")
 else:
result[col] = pd.to_numeric(col_data, downcast='float')
 return result
def read_train() :
df = pd.read_csv(input_file("train.csv"), parse_dates=["timestamp"])
 df.timestamp =(df.timestamp - pd.to_datetime("2016-01-01")).dt.total_seconds() // 3600
return compress_dataframe(df)
 def read_building_metadata() :
return compress_dataframe(pd.read_csv(
 input_file("building_metadata.csv")).fillna(-1)).set_index("building_id")
 site_GMT_offsets = [-5, 0, -7, -5, -8, 0, -5, -5, -5, -6, -7, -5, 0, -6, -5, -5]
def read_weather_train(fix_timestamps=True, interpolate_na=True, add_na_indicators=True):
df = pd.read_csv(input_file("weather_train.csv"), parse_dates=["timestamp"])
 df.timestamp =(df.timestamp - pd.to_datetime("2016-01-01")).dt.total_seconds() // 3600
if fix_timestamps:
GMT_offset_map = {site: offset for site, offset in enumerate(site_GMT_offsets)}
df.timestamp = df.timestamp + df.site_id.map(GMT_offset_map)
 if interpolate_na:
site_dfs = []
for site_id in df.site_id.unique() :
site_df = df[df.site_id == site_id].set_index("timestamp" ).reindex(range(8784))
site_df.site_id = site_id
for col in [c for c in site_df.columns if c != "site_id"]:
if add_na_indicators: site_df[f"had_{col}"] = ~site_df[col].isna()
site_df[col] = site_df[col].interpolate(limit_direction='both', method='linear')
 site_df[col] = site_df[col].fillna(df[col].median())
 site_dfs.append(site_df)
 df = pd.concat(site_dfs ).reset_index()
elif add_na_indicators:
for col in df.columns:
if df[col].isna().any() : df[f"had_{col}"] = ~df[col].isna()
return compress_dataframe(df ).set_index(["site_id", "timestamp"])
 def combined_train_data(fix_timestamps=True, interpolate_na=True, add_na_indicators=True):
Xy = compress_dataframe(read_train().join(read_building_metadata() , on="building_id" ).join(
 read_weather_train(fix_timestamps, interpolate_na, add_na_indicators),
on=["site_id", "timestamp"] ).fillna(-1))
return Xy.drop(columns=["meter_reading"]), Xy.meter_reading
def _add_time_features(X):
return X.assign(tm_day_of_week=(( X.timestamp // 24)% 7), tm_hour_of_day=(X.timestamp % 24))
class CatSplitRegressor(BaseEstimator, RegressorMixin):
def __init__(self, model, col):
self.model = model
self.col = col
def fit(self, X, y):
self.fitted = {}
importances = []
for val in X[self.col].unique() :
X1 = X[X[self.col] == val].drop(columns=[self.col])
 self.fitted[val] = clone(self.model ).fit(X1, y.reindex_like(X1))
importances.append(self.fitted[val].feature_importances_)
 del X1
fi = np.average(importances, axis=0)
 col_index = list(X.columns ).index(self.col)
 self.feature_importances_ = [*fi[:col_index], 0, *fi[col_index:]]
return self
def predict(self, X):
result = np.zeros(len(X))
for val in X[self.col].unique() :
ix = np.nonzero(( X[self.col] == val ).to_numpy())
 predictions = self.fitted[val].predict(X.iloc[ix].drop(columns=[self.col]))
result[ix] = predictions
return result
categorical_columns = [
"building_id", "meter", "site_id", "primary_use", "had_air_temperature", "had_cloud_coverage",
"had_dew_temperature", "had_precip_depth_1_hr", "had_sea_level_pressure", "had_wind_direction",
"had_wind_speed", "tm_day_of_week", "tm_hour_of_day"
]
class LGBMWrapper(BaseEstimator, RegressorMixin):
def __init__(self, categorical_feature=None, **params):
self.model = LGBMRegressor(**params)
 self.categorical_feature = categorical_feature
def fit(self, X, y):
with warnings.catch_warnings() :
cats = None if self.categorical_feature is None else list(
 X.columns.intersection(self.categorical_feature))
warnings.filterwarnings("ignore",
"categorical_feature in Dataset is overridden".lower())
 self.model.fit(X, y, **({} if cats is None else {"categorical_feature": cats}))
self.feature_importances_ = self.model.feature_importances_
return self
def predict(self, X):
return self.model.predict(X)
 def get_params(self, deep=True):
return {**self.model.get_params(deep), "categorical_feature": self.categorical_feature}
def set_params(self, **params):
ctf = params.pop("categorical_feature", None)
 if ctf is not None: self.categorical_feature = ctf
self.model.set_params(params )<load_from_csv>
data=ImageDataBunch.from_csv(path,
csv_labels='train_labels.csv',
folder='train',
suffix='.tif',
num_workers=2,
ds_tfms=tfms,
bs=64,
size=72,
test=path/'test' ).normalize()
def read_test() :
df = pd.read_csv(input_file("test.csv"), parse_dates=["timestamp"])
 df.timestamp =(df.timestamp - pd.to_datetime("2016-01-01")).dt.total_seconds() // 3600
return compress_dataframe(df ).set_index("row_id")
 def read_weather_test(fix_timestamps=True, interpolate_na=True, add_na_indicators=True):
df = pd.read_csv(input_file("weather_test.csv"), parse_dates=["timestamp"])
 df.timestamp =(df.timestamp - pd.to_datetime("2016-01-01")).dt.total_seconds() // 3600
if fix_timestamps:
GMT_offset_map = {site: offset for site, offset in enumerate(site_GMT_offsets)}
df.timestamp = df.timestamp + df.site_id.map(GMT_offset_map)
 if interpolate_na:
site_dfs = []
for site_id in df.site_id.unique() :
site_df = df[df.site_id == site_id].set_index("timestamp" ).reindex(range(8784, 26304))
site_df.site_id = site_id
for col in [c for c in site_df.columns if c != "site_id"]:
if add_na_indicators: site_df[f"had_{col}"] = ~site_df[col].isna()
site_df[col] = site_df[col].interpolate(limit_direction='both', method='linear')
 site_df[col] = site_df[col].fillna(df[col].median())
 site_dfs.append(site_df)
 df = pd.concat(site_dfs ).reset_index()
elif add_na_indicators:
for col in df.columns:
if df[col].isna().any() : df[f"had_{col}"] = ~df[col].isna()
return compress_dataframe(df ).set_index(["site_id", "timestamp"])
 def combined_test_data(fix_timestamps=True, interpolate_na=True, add_na_indicators=True):
X = compress_dataframe(read_test().join(read_building_metadata() , on="building_id" ).join(
 read_weather_test(fix_timestamps, interpolate_na, add_na_indicators),
on=["site_id", "timestamp"] ).fillna(-1))
return X<save_to_csv>
data.show_batch(rows=3,figsize=(8,10))
X, y = combined_train_data()
bad_rows = find_bad_rows(X, y)
 pd.Series(bad_rows.sort_values() ).to_csv("rows_to_drop.csv", header=False, index=False )<train_on_grid>
learner= create_cnn(data, models.resnet50, metrics=[accuracy], model_dir="/tmp/models/" )
X = X.drop(index=bad_rows)
 y = y.reindex_like(X)
 X = compress_dataframe(_add_time_features(X))
X = X.drop(columns="timestamp")
 y = np.log1p(y)
 model = CatSplitRegressor(
 LGBMWrapper(random_state=0, n_jobs=-1, categorical_feature=categorical_columns), "meter")
 model.fit(X, y)
 del X, y<prepare_output>
learner.fit_one_cycle(6, max_lr=(1e-4, 1e-3, 1e-2), wd=(1e-4, 1e-4, 1e-1))
X = combined_test_data()
X = compress_dataframe(_add_time_features(X))
X = X.drop(columns="timestamp")
 predictions = pd.DataFrame({
"row_id": X.index,
"meter_reading": np.clip(np.expm1(model.predict(X)) , 0, None)
 })
 del X<save_to_csv>
learner.unfreeze()
predictions.to_csv("submission.csv", index=False, float_format="%.4f" )<install_modules>
learner.fit_one_cycle(14, slice(1e-5,1e-4,1e-3))
! pip install pytorch_pretrained_bert pytorch-nlp -q<set_options>
learner.save('stage-1' )
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 device<load_from_disk>
pred_data= ImageDataBunch.from_csv(path,
csv_labels='train_labels.csv',
folder='train',
suffix='.tif',
num_workers=2,
ds_tfms=get_transforms(flip_vert=True, max_warp=0),
bs=64,
test=path/'test' ).normalize(imagenet_stats )
!unzip -o '/kaggle/input/jigsaw-toxic-comment-classification-challenge/*.zip' -d /kaggle/working > /dev/null<load_from_csv>
predictor=create_cnn(pred_data, models.resnet50, metrics=[accuracy], model_dir="/tmp/models/" ).load('stage-1' )
df_train = pd.read_csv("train.csv")
 df_test = pd.read_csv("test.csv")
 sample_submission = pd.read_csv("sample_submission.csv")
 print(df_train.shape, df_test.shape, sample_submission.shape)
 df_train.head()<feature_engineering>
pred_class,pred_idx,outputs = predictor.predict(x)
 pred_class
def strip(text):
soup = BeautifulSoup(text, 'html.parser')
 text = re.sub('\[[^]]*\]', '', soup.get_text())
 pattern=r"[^a-zA-z0-9\s,']"
text=re.sub(pattern,'',text)
 return text
df_train["comment_text"] = df_train["comment_text"].apply(strip)
 df_test["comment_text"] = df_test["comment_text"].apply(strip )<define_variables>
preds_val, y_val=learner.get_preds()
train_sentences = df_train["comment_text"]
test_sentences = df_test["comment_text"]
train_sentences = ["[CLS] "+ i + " [SEP]"for i in train_sentences]
test_sentences = ["[CLS] "+ i + " [SEP]"for i in test_sentences]
train_sentences[0], test_sentences[0]<string_transform>
fpr, tpr, thresholds = roc_curve(y_val.numpy() , preds_val.numpy() [:,1], pos_label=1)
 pred_score = auc(fpr, tpr)
 print(f'ROC area is {pred_score}' )
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case = True)
 train_tokenizer_texts = list(map(lambda t: tokenizer.tokenize(t)[:510], tqdm(train_sentences)))
 test_tokenizer_texts = list(map(lambda t: tokenizer.tokenize(t)[:510], tqdm(test_sentences)))
 np.array(train_tokenizer_texts[0]), np.array(test_tokenizer_texts[0] )<data_type_conversions>
preds_test,y_test = learner.TTA(ds_type=DatasetType.Test)
labels = df_train[["toxic","severe_toxic","obscene","threat","insult","identity_hate"]].to_numpy()<string_transform>
sub=pd.read_csv(f'{path}/sample_submission.csv' ).set_index('id')
 sub.head()
MAX_LEN = 128
input_ids = [tokenizer.convert_tokens_to_ids(x)for x in tqdm(train_tokenizer_texts)]
input_ids = pad_sequences(sequences = input_ids, maxlen = MAX_LEN, dtype = 'long', padding='post', truncating='post')
 test_input_ids = [tokenizer.convert_tokens_to_ids(x)for x in tqdm(test_tokenizer_texts)]
test_input_ids = pad_sequences(sequences = test_input_ids, maxlen = MAX_LEN, dtype = 'long', padding='post', truncating='post')
 input_ids[0], test_input_ids[0]<categorify>
names=np.vectorize(lambda img_name: str(img_name ).split('/')[-1][:-4])
 file_names= names(data.test_ds.items ).astype(str )
<split><EOS> 
sub.loc[file_names,'label']=preds_test.numpy() [:,1]
sub.to_csv(f'submission_{pred_score}.csv' )