Kaggle competition: Statoil/C-CORE Iceberg Classifier Challenge
warnings.filterwarnings("ignore", category=FutureWarning)
<load_pretrained>
sub_path = ".. /input/statoil-iceberg-submissions"
all_files = os.listdir(sub_path)
 outs = [pd.read_csv(os.path.join(sub_path, f), index_col=0)for f in all_files]
concat_sub = pd.concat(outs, axis=1)
 cols = list(map(lambda x: "is_iceberg_" + str(x), range(len(concat_sub.columns))))
concat_sub.columns = cols
concat_sub.reset_index(inplace=True)
 concat_sub.head()
nltk.download("wordnet" )<set_options>
concat_sub['is_iceberg_max'] = concat_sub.iloc[:, 1:6].max(axis=1)
 concat_sub['is_iceberg_min'] = concat_sub.iloc[:, 1:6].min(axis=1)
 concat_sub['is_iceberg_mean'] = concat_sub.iloc[:, 1:6].mean(axis=1)
 concat_sub['is_iceberg_median'] = concat_sub.iloc[:, 1:6].median(axis=1 )
%config IPCompleter.greedy=True
%matplotlib inline
%reload_ext nb_black<define_variables>
cutoff_lo = 0.7
cutoff_hi = 0.3
curr_dir = ".. /input"
already_trained = False
train_file = os.path.join(curr_dir, "nlp-getting-started", "train.csv")
 test_file = os.path.join(curr_dir, "nlp-getting-started", "test.csv")
 sample_submission_file = os.path.join(
 curr_dir, "nlp-getting-started", "sample_submission.csv"
)<set_options>
concat_sub['is_iceberg'] = concat_sub['is_iceberg_mean']
concat_sub[['id', 'is_iceberg']].to_csv('stack_mean.csv',
index=False, float_format='%.6f' )
params = {
"legend.fontsize": "x-large",
"figure.figsize":(16, 5),
"axes.labelsize": "x-large",
"axes.titlesize": "x-large",
"xtick.labelsize": "x-large",
"ytick.labelsize": "x-large",
}
pylab.rcParams.update(params )<load_from_csv>
concat_sub['is_iceberg'] = concat_sub['is_iceberg_median']
concat_sub[['id', 'is_iceberg']].to_csv('stack_median.csv',
index=False, float_format='%.6f' )
df = pd.read_csv(train_file)
 df.head()<count_missing_values>
concat_sub['is_iceberg'] = np.where(np.all(concat_sub.iloc[:,1:6] > cutoff_lo, axis=1), 1,
np.where(np.all(concat_sub.iloc[:,1:6] < cutoff_hi, axis=1),
0, concat_sub['is_iceberg_median']))
concat_sub[['id', 'is_iceberg']].to_csv('stack_pushout_median.csv',
index=False, float_format='%.6f' )
df.isnull().sum()<count_duplicates>
concat_sub['is_iceberg'] = np.where(np.all(concat_sub.iloc[:,1:6] > cutoff_lo, axis=1),
concat_sub['is_iceberg_max'],
np.where(np.all(concat_sub.iloc[:,1:6] < cutoff_hi, axis=1),
concat_sub['is_iceberg_min'],
concat_sub['is_iceberg_mean']))
concat_sub[['id', 'is_iceberg']].to_csv('stack_minmax_mean.csv',
index=False, float_format='%.6f' )
text_data_duplicate_map = df.text.duplicated().value_counts()
print(text_data_duplicate_map)
 plotBarGraph(
 text_data_duplicate_map,
"Duplicate",
"Frequency",
"Text Data Duplicate or Not",
switch_axis=True,
)<count_duplicates>
concat_sub['is_iceberg'] = np.where(np.all(concat_sub.iloc[:,1:6] > cutoff_lo, axis=1),
concat_sub['is_iceberg_max'],
np.where(np.all(concat_sub.iloc[:,1:6] < cutoff_hi, axis=1),
concat_sub['is_iceberg_min'],
concat_sub['is_iceberg_median']))
concat_sub['is_iceberg'] = np.clip(concat_sub['is_iceberg'].values, 0.001, 0.999)
 concat_sub[['id', 'is_iceberg']].to_csv('stack_minmax_median.csv',
index=False, float_format='%.6f' )
print("Original dataframe shape: ", df.shape)
 df.drop_duplicates(subset="text", keep="first", inplace=True)
 print("Dropping duplicate rows and keeping original value shape: ", df.shape)
 df.text.duplicated().value_counts()<filter>
sub_base = pd.read_csv('.. /input/statoil-iceberg-submissions/sub_200_ens_densenet.csv' )
def search_text_data(query, column="text"):
return df[df[column].str.contains(query)][column]
print(search_text_data("volcano")[:5] )<string_transform>
concat_sub['is_iceberg_base'] = sub_base['is_iceberg']
concat_sub['is_iceberg'] = np.where(np.all(concat_sub.iloc[:,1:6] > cutoff_lo, axis=1),
concat_sub['is_iceberg_max'],
np.where(np.all(concat_sub.iloc[:,1:6] < cutoff_hi, axis=1),
concat_sub['is_iceberg_min'],
concat_sub['is_iceberg_base']))
concat_sub['is_iceberg'] = np.clip(concat_sub['is_iceberg'].values, 0.001, 0.999)
 concat_sub[['id', 'is_iceberg']].to_csv('stack_minmax_bestbase.csv',
index=False, float_format='%.6f' )