Kaggle competition: University of Liverpool - Ion Switching
<SOS> metric: MacroFScore Kaggle data source: university-of-liverpool-ion-switching<create_dataframe>
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
test_feats = pd.DataFrame.from_dict(features, orient='index')
 test_feats.columns = [f'pic_{i}' for i in range(test_feats.shape[1])]<rename_columns>
seed = 42
test_size = 0.2
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 workspace = "./unet"
backbone = [1, 1, 1, 1]
encoder_channels = np.array([64, 128, 256, 512, 1024])*2
decoder_channels = np.array([512, 256, 128, 64])*2
fold = 0
time_step = 4000
time_step_test = 10000
stride = 2
batch_size = 10
TRAIN = True
PREDICT = True
ENSEMBLE = False
np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 print(device )
train_feats = train_feats.reset_index()
train_feats.rename({'index': 'PetID'}, axis='columns', inplace=True)
 test_feats = test_feats.reset_index()
test_feats.rename({'index': 'PetID'}, axis='columns', inplace=True )<concatenate>
train = pd.read_csv('/kaggle/input/liverpool-ion-switching/train.csv')
 test = pd.read_csv('/kaggle/input/liverpool-ion-switching/test.csv')
 sample = pd.read_csv('/kaggle/input/liverpool-ion-switching/sample_submission.csv' )
all_ids = pd.concat([train, test], axis=0, ignore_index=True, sort=False)[['PetID']]
all_ids.shape<concatenate>
def make_batch(df, batchsize=500000):
batches = df.shape[0] // batchsize
df['batch'] = 0
for i in range(batches):
idx = np.arange(i*batchsize,(i+1)*batchsize)
 df.loc[idx, 'batch'] = i + 1
return df
n_components = 32
svd_ = TruncatedSVD(n_components=n_components, random_state=1337)
 features_df = pd.concat([train_feats, test_feats], axis=0)
 features = features_df[[f'pic_{i}' for i in range(256)]].values
svd_col = svd_.fit_transform(features)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('IMG_SVD_')
 img_features = pd.concat([all_ids, svd_col], axis=1 )<load_from_csv>
train = make_batch(train)
 test = make_batch(test, batchsize=500000)
 train.groupby('batch')['signal'].describe()
labels_breed = pd.read_csv('.. /input/petfinder-adoption-prediction/breed_labels.csv')
 labels_state = pd.read_csv('.. /input/petfinder-adoption-prediction/color_labels.csv')
 labels_color = pd.read_csv('.. /input/petfinder-adoption-prediction/state_labels.csv' )<define_variables>
train_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_images/*.jpg'))
train_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_metadata/*.json'))
train_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_sentiment/*.json'))
print(f'num of train images files: {len(train_image_files)}')
 print(f'num of train metadata files: {len(train_metadata_files)}')
 print(f'num of train sentiment files: {len(train_sentiment_files)}')
 test_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_images/*.jpg'))
test_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_metadata/*.json'))
test_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_sentiment/*.json'))
print(f'num of test images files: {len(test_image_files)}')
 print(f'num of test metadata files: {len(test_metadata_files)}')
 print(f'num of test sentiment files: {len(test_sentiment_files)}' )<load_pretrained>
train_segm_separators = np.concatenate([[0,500000,600000], np.arange(1000000,5000000+1,500000)])
 train_segm_signal_groups = [0,0,0,1,2,4,3,1,2,3,4]
train_segm_is_shifted = [False, True, False, False, False, False, False, True, True, True, True]
train_signal = np.split(train['signal'].values, train_segm_separators[1:-1])
 train_opench = np.split(train['open_channels'].values, train_segm_separators[1:-1] )
class PetFinderParser(object):
def __init__(self, debug=False):
self.debug = debug
self.sentence_sep = ' '
self.extract_sentiment_text = False
def open_json_file(self, filename):
with open(filename, 'r', encoding='utf-8')as f:
json_file = json.load(f)
 return json_file
def parse_sentiment_file(self, file):

 file_sentiment = file['documentSentiment']
file_entities = [x['name'] for x in file['entities']]
file_entities = self.sentence_sep.join(file_entities)
 file_sentences_sentiment = [x['sentiment'] for x in file['sentences']]
file_sentences_sentiment = pd.DataFrame.from_dict(
 file_sentences_sentiment, orient='columns')
 file_sentences_sentiment_df = pd.DataFrame(
 {
'magnitude_sum': file_sentences_sentiment['magnitude'].sum(axis=0),
'score_sum': file_sentences_sentiment['score'].sum(axis=0),
'magnitude_mean': file_sentences_sentiment['magnitude'].mean(axis=0),
'score_mean': file_sentences_sentiment['score'].mean(axis=0),
'magnitude_var': file_sentences_sentiment['magnitude'].var(axis=0),
'score_var': file_sentences_sentiment['score'].var(axis=0),
'magnitude_std': file_sentences_sentiment['magnitude'].std(axis=0),
'score_std': file_sentences_sentiment['score'].std(axis=0),
}, index=[0] 
)
 df_sentiment = pd.DataFrame.from_dict(file_sentiment, orient='index' ).T
df_sentiment = pd.concat([df_sentiment, file_sentences_sentiment_df], axis=1)
 df_sentiment['entities'] = file_entities
df_sentiment = df_sentiment.add_prefix('sentiment_')
 return df_sentiment
def parse_metadata_file(self, file):

 file_keys = list(file.keys())
 if 'labelAnnotations' in file_keys:
file_annots = file['labelAnnotations']
file_top_score = np.asarray([x['score'] for x in file_annots] ).mean()
file_top_desc = [x['description'] for x in file_annots]
else:
file_top_score = np.nan
file_top_desc = ['']
file_colors = file['imagePropertiesAnnotation']['dominantColors']['colors']
file_crops = file['cropHintsAnnotation']['cropHints']
file_color_score = np.asarray([x['score'] for x in file_colors] ).mean()
file_color_pixelfrac = np.asarray([x['pixelFraction'] for x in file_colors] ).mean()
file_crop_conf = np.asarray([x['confidence'] for x in file_crops] ).mean()
if 'importanceFraction' in file_crops[0].keys() :
file_crop_importance = np.asarray([x['importanceFraction'] for x in file_crops] ).mean()
else:
file_crop_importance = np.nan
df_metadata = {
'annots_score': file_top_score,
'color_score': file_color_score,
'color_pixelfrac': file_color_pixelfrac,
'crop_conf': file_crop_conf,
'crop_importance': file_crop_importance,
'annots_top_desc': self.sentence_sep.join(file_top_desc)
 }
df_metadata = pd.DataFrame.from_dict(df_metadata, orient='index' ).T
df_metadata = df_metadata.add_prefix('metadata_')
 return df_metadata
def extract_additional_features(pet_id, mode='train'):
sentiment_filename = f'.. /input/petfinder-adoption-prediction/{mode}_sentiment/{pet_id}.json'
try:
sentiment_file = pet_parser.open_json_file(sentiment_filename)
 df_sentiment = pet_parser.parse_sentiment_file(sentiment_file)
 df_sentiment['PetID'] = pet_id
except FileNotFoundError:
df_sentiment = []
dfs_metadata = []
metadata_filenames = sorted(glob.glob(f'.. /input/petfinder-adoption-prediction/{mode}_metadata/{pet_id}*.json'))
if len(metadata_filenames)> 0:
for f in metadata_filenames:
metadata_file = pet_parser.open_json_file(f)
 df_metadata = pet_parser.parse_metadata_file(metadata_file)
 df_metadata['PetID'] = pet_id
dfs_metadata.append(df_metadata)
 dfs_metadata = pd.concat(dfs_metadata, ignore_index=True, sort=False)
 dfs = [df_sentiment, dfs_metadata]
return dfs
pet_parser = PetFinderParser()<categorify>
test_segm_separators = np.concatenate([np.arange(0,1000000+1,100000), [1500000,2000000]])
 test_segm_signal_groups = [0,2,3,0,1,4,3,4,0,2,0,0]
test_segm_is_shifted = [True, True, False, False, True, False, True, True, True, False, True, False]
test_signal = np.split(test['signal'].values, test_segm_separators[1:-1] )
aggregates = ['sum', 'mean', 'var', 'std']
sent_agg = ['sum']
train_metadata_desc = train_dfs_metadata.groupby(['PetID'])['metadata_annots_top_desc'].unique()
train_metadata_desc = train_metadata_desc.reset_index()
train_metadata_desc[
'metadata_annots_top_desc'] = train_metadata_desc[
'metadata_annots_top_desc'].apply(lambda x: ' '.join(x))
prefix = 'metadata'
train_metadata_gr = train_dfs_metadata.drop(['metadata_annots_top_desc'], axis=1)
 for i in train_metadata_gr.columns:
if 'PetID' not in i:
train_metadata_gr[i] = train_metadata_gr[i].astype(float)
 train_metadata_gr = train_metadata_gr.groupby(['PetID'] ).agg(aggregates)
 train_metadata_gr.columns = pd.Index([f'{c[0]}_{c[1].upper() }' for c in train_metadata_gr.columns.tolist() ])
 train_metadata_gr = train_metadata_gr.reset_index()
train_sentiment_desc = train_dfs_sentiment.groupby(['PetID'])['sentiment_entities'].unique()
train_sentiment_desc = train_sentiment_desc.reset_index()
train_sentiment_desc[
'sentiment_entities'] = train_sentiment_desc[
'sentiment_entities'].apply(lambda x: ' '.join(x))
prefix = 'sentiment'
train_sentiment_gr = train_dfs_sentiment.drop(['sentiment_entities'], axis=1)
 for i in train_sentiment_gr.columns:
if 'PetID' not in i:
train_sentiment_gr[i] = train_sentiment_gr[i].astype(float)
 train_sentiment_gr = train_sentiment_gr.groupby(['PetID'] ).agg(sent_agg)
 train_sentiment_gr.columns = pd.Index([f'{c[0]}' for c in train_sentiment_gr.columns.tolist() ])
 train_sentiment_gr = train_sentiment_gr.reset_index()
test_metadata_desc = test_dfs_metadata.groupby(['PetID'])['metadata_annots_top_desc'].unique()
test_metadata_desc = test_metadata_desc.reset_index()
test_metadata_desc[
'metadata_annots_top_desc'] = test_metadata_desc[
'metadata_annots_top_desc'].apply(lambda x: ' '.join(x))
prefix = 'metadata'
test_metadata_gr = test_dfs_metadata.drop(['metadata_annots_top_desc'], axis=1)
 for i in test_metadata_gr.columns:
if 'PetID' not in i:
test_metadata_gr[i] = test_metadata_gr[i].astype(float)
 test_metadata_gr = test_metadata_gr.groupby(['PetID'] ).agg(aggregates)
 test_metadata_gr.columns = pd.Index([f'{c[0]}_{c[1].upper() }' for c in test_metadata_gr.columns.tolist() ])
 test_metadata_gr = test_metadata_gr.reset_index()
test_sentiment_desc = test_dfs_sentiment.groupby(['PetID'])['sentiment_entities'].unique()
test_sentiment_desc = test_sentiment_desc.reset_index()
test_sentiment_desc[
'sentiment_entities'] = test_sentiment_desc[
'sentiment_entities'].apply(lambda x: ' '.join(x))
prefix = 'sentiment'
test_sentiment_gr = test_dfs_sentiment.drop(['sentiment_entities'], axis=1)
 for i in test_sentiment_gr.columns:
if 'PetID' not in i:
test_sentiment_gr[i] = test_sentiment_gr[i].astype(float)
 test_sentiment_gr = test_sentiment_gr.groupby(['PetID'] ).agg(sent_agg)
 test_sentiment_gr.columns = pd.Index([f'{c[0]}' for c in test_sentiment_gr.columns.tolist() ])
 test_sentiment_gr = test_sentiment_gr.reset_index()<merge>
train_signal_shift_clean = []
train_signal_detrend = []
for data, use_fit, signal in zip(train_signal_shift, train_segm_is_shifted, train_signal):
if use_fit:
data_x = np.arange(len(data), dtype=float)* window_size + window_size/2
fit = np.flip(np.polyfit(data_x, data, 4))
data_x = np.arange(len(data)* window_size, dtype=float)
 data_2 = np.sum([ c * data_x ** i for i, c in enumerate(fit)], axis=0)
 else:
data_2 = np.zeros(len(data)* window_size, dtype=float)
 train_signal_detrend.append(signal + data_2 )
train_proc = train.copy()
train_proc = train_proc.merge(
 train_sentiment_gr, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_metadata_gr, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_metadata_desc, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_sentiment_desc, how='left', on='PetID')
 test_proc = test.copy()
test_proc = test_proc.merge(
 test_sentiment_gr, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_metadata_gr, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_metadata_desc, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_sentiment_desc, how='left', on='PetID')
 print(train_proc.shape, test_proc.shape)
 assert train_proc.shape[0] == train.shape[0]
assert test_proc.shape[0] == test.shape[0]<merge>
train_signal = np.ndarray(0)
 for arr in train_signal_detrend:
train_signal = np.append(train_signal, arr )
train_breed_main = train_proc[['Breed1']].merge(
 labels_breed, how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
train_breed_main = train_breed_main.iloc[:, 2:]
train_breed_main = train_breed_main.add_prefix('main_breed_')
 train_breed_second = train_proc[['Breed2']].merge(
 labels_breed, how='left',
left_on='Breed2', right_on='BreedID',
suffixes=('', '_second_breed'))
train_breed_second = train_breed_second.iloc[:, 2:]
train_breed_second = train_breed_second.add_prefix('second_breed_')
 train_proc = pd.concat(
 [train_proc, train_breed_main, train_breed_second], axis=1)
 test_breed_main = test_proc[['Breed1']].merge(
 labels_breed, how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
test_breed_main = test_breed_main.iloc[:, 2:]
test_breed_main = test_breed_main.add_prefix('main_breed_')
 test_breed_second = test_proc[['Breed2']].merge(
 labels_breed, how='left',
left_on='Breed2', right_on='BreedID',
suffixes=('', '_second_breed'))
test_breed_second = test_breed_second.iloc[:, 2:]
test_breed_second = test_breed_second.add_prefix('second_breed_')
 test_proc = pd.concat(
 [test_proc, test_breed_main, test_breed_second], axis=1)
 print(train_proc.shape, test_proc.shape )<concatenate>
test_signal_shift_clean = []
test_signal_detrend = []
test_remove_shift = [True, True, False, False, True, False, True, True, True, False, True, False]
for data, use_fit, signal in zip(test_signal_shift, test_segm_is_shifted, test_signal):
if use_fit:
data_x = np.arange(len(data), dtype=float)* window_size + window_size/2
fit = np.flip(np.polyfit(data_x, data, 4))
data_x = np.arange(len(data)* window_size, dtype=float)
 data_2 = np.sum([ c * data_x ** i for i, c in enumerate(fit)], axis=0)
 else:
data_2 = np.zeros(len(data)* window_size, dtype=float)
 test_signal_detrend.append(signal + data_2 )
X = pd.concat([train_proc, test_proc], ignore_index=True, sort=False )<define_variables>
test_signal = np.ndarray(0)
 for arr in test_signal_detrend:
test_signal = np.append(test_signal, arr )
X_temp = X.copy()
text_columns = ['Description', 'metadata_annots_top_desc', 'sentiment_entities']
categorical_columns = ['main_breed_BreedName', 'second_breed_BreedName']
to_drop_columns = ['PetID', 'Name', 'RescuerID']<merge>
train_time = train['time'][:].values
test_time = test['time'][:].values
rescuer_count = X.groupby(['RescuerID'])['PetID'].count().reset_index()
rescuer_count.columns = ['RescuerID', 'RescuerID_COUNT']
X_temp = X_temp.merge(rescuer_count, how='left', on='RescuerID' )<feature_engineering>
train = train.drop(['signal'], axis=1)
 test = test.drop(['signal'], axis=1 )
for i in categorical_columns:
X_temp.loc[:, i] = pd.factorize(X_temp.loc[:, i])[0]<feature_engineering>
def data_filter(data, time, signal, sigm=6, batchsize=500000):
mean_std, cut_off, lower, upper, filtered_signal, filtered_time = [], [], [], [], [], []
for x, y in zip(data.groupby('batch')['signal'].agg('mean'), data.groupby('batch')['signal'].agg('std')) :
mean_std.append(( x, y))
for i in range(len(mean_std)) :
lower.append(mean_std[i][0] -(mean_std[i][1])*sigm)
 upper.append(mean_std[i][0] +(mean_std[i][1])*sigm)
 for i in range(len(data['batch'].value_counts())) :
for j in range(data['batch'].value_counts() [1]):
if(signal[j+i*batchsize] < upper[i])and(signal[j+i*batchsize] > lower[i]):
filtered_signal.append(signal[j+i*batchsize])
 filtered_time.append(time[j+i*batchsize])
 return filtered_time, filtered_signal
X_text = X_temp[text_columns]
for i in X_text.columns:
X_text.loc[:, i] = X_text.loc[:, i].fillna('none' )<categorify>
train['signal'] = train_signal
test['signal'] = test_signal
X_temp['Length_Description'] = X_text['Description'].map(len)
 X_temp['Length_metadata_annots_top_desc'] = X_text['metadata_annots_top_desc'].map(len)
 X_temp['Lengths_sentiment_entities'] = X_text['sentiment_entities'].map(len )<feature_engineering>
train_time, train_signal = data_filter(train, train_time, train_signal)
 test_time, test_signal = data_filter(test, test_time, test_signal, sigm=6, batchsize=500000 )
n_components = 16
text_features = []
for i in X_text.columns:
print(f'generating features from: {i}')
 tfv = TfidfVectorizer(min_df=2, max_features=None,
strip_accents='unicode', analyzer='word', token_pattern=r'(?u)\b\w+\b',
ngram_range=(1, 3), use_idf=1, smooth_idf=1, sublinear_tf=1)
 svd_ = TruncatedSVD(
 n_components=n_components, random_state=1337)
 tfidf_col = tfv.fit_transform(X_text.loc[:, i].values)
 svd_col = svd_.fit_transform(tfidf_col)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('TFIDF_{}_'.format(i))
text_features.append(svd_col)
 text_features = pd.concat(text_features, axis=1)
 X_temp = pd.concat([X_temp, text_features], axis=1)
 for i in X_text.columns:
X_temp = X_temp.drop(i, axis=1 )<merge>
def interpol(time, signal):
if len(time)> 2000000:
LENA =(0.0001, 500.0001, 0.0001)
 elif(len(time)> 500000)and(len(time)< 2000000):
LENA =(500.0001, 700.0001, 0.0001)
 else:
LENA =(350.0001, 400.0001, 0.0001)
 y = np.array(signal[:])
 x = np.array(time[:])
 f = interpolate.interp1d(x, y)
 new_time = np.arange(*LENA)
 new_signal = f(new_time)
 return new_time, new_signal
X_temp = X_temp.merge(img_features, how='left', on='PetID' )<define_variables>
train = train.drop(['signal', 'time'], axis=1)
 test = test.drop(['signal', 'time'], axis=1)
 train['time'], train_signal = interpol(train_time, train_signal)
 test['time'], test_signal = interpol(test_time, test_signal)
 train['signal'] = train_signal
test['signal'] = test_signal
train_df_ids = train[['PetID']]
test_df_ids = test[['PetID']]
train_df_imgs = pd.DataFrame(train_image_files)
 train_df_imgs.columns = ['image_filename']
train_imgs_pets = train_df_imgs['image_filename'].apply(lambda x: x.split(split_char)[-1].split('-')[0])
 test_df_imgs = pd.DataFrame(test_image_files)
 test_df_imgs.columns = ['image_filename']
test_imgs_pets = test_df_imgs['image_filename'].apply(lambda x: x.split(split_char)[-1].split('-')[0])
 train_df_imgs = train_df_imgs.assign(PetID=train_imgs_pets)
 test_df_imgs = test_df_imgs.assign(PetID=test_imgs_pets)
 def getSize(filename):
st = os.stat(filename)
 return st.st_size
def getDimensions(filename):
img_size = Image.open(filename ).size
return img_size
train_df_imgs['image_size'] = train_df_imgs['image_filename'].apply(getSize)
 train_df_imgs['temp_size'] = train_df_imgs['image_filename'].apply(getDimensions)
 train_df_imgs['width'] = train_df_imgs['temp_size'].apply(lambda x : x[0])
 train_df_imgs['height'] = train_df_imgs['temp_size'].apply(lambda x : x[1])
 train_df_imgs = train_df_imgs.drop(['temp_size'], axis=1)
 test_df_imgs['image_size'] = test_df_imgs['image_filename'].apply(getSize)
 test_df_imgs['temp_size'] = test_df_imgs['image_filename'].apply(getDimensions)
 test_df_imgs['width'] = test_df_imgs['temp_size'].apply(lambda x : x[0])
 test_df_imgs['height'] = test_df_imgs['temp_size'].apply(lambda x : x[1])
 test_df_imgs = test_df_imgs.drop(['temp_size'], axis=1)
 aggs = {
'image_size': ['sum', 'mean', 'var', 'std'],
'width': ['sum', 'mean', 'var', 'std'],
'height': ['sum', 'mean', 'var', 'std'],
}
agg_train_imgs = train_df_imgs.groupby('PetID' ).agg(aggs)
 new_columns = [
k + '_' + agg for k in aggs.keys() for agg in aggs[k]
]
agg_train_imgs.columns = new_columns
agg_train_imgs = agg_train_imgs.reset_index()
agg_test_imgs = test_df_imgs.groupby('PetID' ).agg(aggs)
 new_columns = [
k + '_' + agg for k in aggs.keys() for agg in aggs[k]
]
agg_test_imgs.columns = new_columns
agg_test_imgs = agg_test_imgs.reset_index()
agg_imgs = pd.concat([agg_train_imgs, agg_test_imgs], axis=0 ).reset_index(drop=True )<merge>
from scipy.signal import butter,filtfilt
X_temp = X_temp.merge(agg_imgs, how='left', on='PetID' )<drop_column>
T = 5.0
fs = 30.0
cutoff = 2
nyq = 0.5 * fs
order = 2
n = int(T * fs )
X_temp = X_temp.drop(to_drop_columns, axis=1 )<define_variables>
def butter_lowpass_filter(data, cutoff, fs, order):
normal_cutoff = cutoff / nyq
b, a = butter(order, normal_cutoff, btype='low', analog=False)
 y = filtfilt(b, a, data)
 return y
X_train_non_null = X_train.fillna(-1)
 X_test_non_null = X_test.fillna(-1 )<count_missing_values>
seventh_batch_time, seventh_batch_signal = [x/10000 for x in range(3500001, 4000001, 1)], list(train_signal[3500000:4000000] )
X_train_non_null.isnull().any().any() , X_test_non_null.isnull().any().any()<import_modules>
poper = 0
for i in range(1, 499999):
if(train_signal[i+3500000] > 2.2)or(train_signal[i+3500000] < -3.8):
seventh_batch_time.pop(i-poper)
 seventh_batch_signal.pop(i-poper)
 poper += 1
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):

 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)]
for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):

 if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):

 rater_a = y
rater_b = y_pred
min_rating=None
max_rating=None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j]
/ num_scored_items)
 d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator )<train_on_grid>
_, seventh_batch_signal = interpol(seventh_batch_time, seventh_batch_signal )
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 return -cohen_kappa_score(y, preds, weights='quadratic')
 def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X = X, y = y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 return preds
def coefficients(self):
return self.coef_['x']<init_hyperparams>
train_signal[3500000:4000000] = seventh_batch_signal[:]
xgb_params = {
'eval_metric': 'rmse',
'seed': 1337,
'eta': 0.0123,
'subsample': 0.8,
'colsample_bytree': 0.85,
'tree_method': 'gpu_hist',
'device': 'gpu',
'silent': 1,
}<prepare_x_and_y>
train = train.drop(['signal'], axis=1)
 train['signal'] = train_signal
def run_xgb(params, X_train, X_test):
n_splits = 5
verbose_eval = 1000
num_rounds = 65000
early_stop = 500
kf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=1337)
 oof_train = np.zeros(( X_train.shape[0]))
oof_test = np.zeros(( X_test.shape[0], n_splits))
i = 0
for train_idx, valid_idx in kf.split(X_train, X_train['AdoptionSpeed'].values):
X_tr = X_train.iloc[train_idx, :]
X_val = X_train.iloc[valid_idx, :]
y_tr = X_tr['AdoptionSpeed'].values
X_tr = X_tr.drop(['AdoptionSpeed'], axis=1)
 y_val = X_val['AdoptionSpeed'].values
X_val = X_val.drop(['AdoptionSpeed'], axis=1)
 d_train = xgb.DMatrix(data=X_tr, label=y_tr, feature_names=X_tr.columns)
 d_valid = xgb.DMatrix(data=X_val, label=y_val, feature_names=X_val.columns)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
model = xgb.train(dtrain=d_train, num_boost_round=num_rounds, evals=watchlist,
early_stopping_rounds=early_stop, verbose_eval=verbose_eval, params=params)
 valid_pred = model.predict(xgb.DMatrix(X_val, feature_names=X_val.columns), ntree_limit=model.best_ntree_limit)
 test_pred = model.predict(xgb.DMatrix(X_test, feature_names=X_test.columns), ntree_limit=model.best_ntree_limit)
 oof_train[valid_idx] = valid_pred
oof_test[:, i] = test_pred
i += 1
return model, oof_train, oof_test<train_model>
def do_identity(image, label):
return image, label
def do_flip(image, label):
image = np.ascontiguousarray(np.flip(image, axis=1))
label = np.ascontiguousarray(np.flip(label))
return image, label
def train_augment(image, label):
for op in np.random.choice([
lambda image, label: do_identity(image, label),
lambda image, label: do_flip(image, label),
], 1):
image, label = op(image, label)
 return image, label
def valid_augment(image, label):
return image, label
model, oof_train, oof_test = run_xgb(xgb_params, X_train_non_null, X_test_non_null )<compute_train_metric>
class IonDataset(Dataset):
def __init__(self, data, labels=None, type='train', transform=None):
self.data = data
self.labels = labels
self.type = type
self.transform = transform
def __getitem__(self, i):
signal = self.data[i].astype(np.float32)
 if self.type == 'train':
label = self.labels[i].astype(np.int64)
 if self.transform is not None:
signal, label = self.transform(signal, label)
 return signal, label
else:
return signal
def __len__(self):
return len(self.data )
optR = OptimizedRounder()
optR.fit(oof_train, X_train['AdoptionSpeed'].values)
 coefficients = optR.coefficients()
valid_pred = optR.predict(oof_train, coefficients)
 qwk = quadratic_weighted_kappa(X_train['AdoptionSpeed'].values, valid_pred)
 print("QWK = ", qwk )<predict_on_test>
class SEModule(nn.Module):
def __init__(self, in_channels, reduction=4):
super().__init__()
self.conv1 = nn.Conv1d(in_channels, in_channels//reduction, kernel_size=1, padding=0)
 self.conv2 = nn.Conv1d(in_channels//reduction, in_channels, kernel_size=1, padding=0)
 def forward(self, x):
s = F.adaptive_avg_pool1d(x, 1)
 s = self.conv1(s)
 s = F.relu(s, inplace=True)
 s = self.conv2(s)
 x = x + torch.sigmoid(s)
 return x
class ConvBR1d(nn.Module):
def __init__(self, in_channels, out_channels, kernel_size=1, padding=0, dilation=1, stride=1, groups=1, is_activation=True):
super().__init__()
self.conv = nn.Conv1d(in_channels, out_channels, kernel_size=kernel_size, padding=padding, dilation=dilation, stride=stride, groups=groups, bias=False)
 self.bn = nn.BatchNorm1d(out_channels)
 self.is_activation = is_activation
if is_activation:
self.relu = nn.ReLU(inplace=True)
 def forward(self, x):
x = self.bn(self.conv(x))
if self.is_activation:
x = self.relu(x)
 return x
class SENextBottleneck(nn.Module):
def __init__(self, in_channels, out_channels, stride=1, groups=32, reduction=16, pool=None, is_shortcut=False):
super().__init__()
mid_channels = out_channels // 2
self.conv1 = ConvBR1d(in_channels, mid_channels, 1, 0, 1,)
 self.conv2 = ConvBR1d(mid_channels, mid_channels, 3, 1, 1, groups=groups)
 self.conv3 = ConvBR1d(mid_channels, out_channels, 1, 0, 1, is_activation=False)
 self.se = SEModule(out_channels, reduction)
 self.stride = stride
self.is_shortcut = is_shortcut
if is_shortcut:
self.shortcut = ConvBR1d(in_channels, out_channels, 1, 0, 1, is_activation=False)
 if stride > 1:
if pool == 'max':
self.pool = nn.MaxPool1d(stride, stride)
 elif pool == 'avg':
self.pool = nn.AvgPool1d(stride, stride)
 def forward(self, x):
s = self.conv1(x)
 s = self.conv2(s)
 if self.stride > 1:
s = self.pool(s)
 s = self.conv3(s)
 s = self.se(s)
 if self.is_shortcut:
if self.stride > 1:
x = F.avg_pool1d(x, self.stride, self.stride)
 x = self.shortcut(x)
 x = x + s
x = F.relu(x, inplace=True)
 return x
class Encoder(nn.Module):
def __init__(self, num_features=1):
super().__init__()
self.block0 = nn.Sequential(
 ConvBR1d(num_features, encoder_channels[0], kernel_size=5, stride=1, padding=2),
ConvBR1d(encoder_channels[0], encoder_channels[0], kernel_size=3, stride=1, padding=1),
ConvBR1d(encoder_channels[0], encoder_channels[0], kernel_size=3, stride=1, padding=1), 
)
 self.block1 = nn.Sequential(
 SENextBottleneck(encoder_channels[0], encoder_channels[1], stride=stride, is_shortcut=True, pool='max'),
*[SENextBottleneck(encoder_channels[1], encoder_channels[1], stride=1, is_shortcut=False)for i in range(backbone[0])] 
)
 self.block2 = nn.Sequential(
 SENextBottleneck(encoder_channels[1], encoder_channels[2], stride=stride, is_shortcut=True, pool='max'),
*[SENextBottleneck(encoder_channels[2], encoder_channels[2], stride=1, is_shortcut=False)for i in range(backbone[1])] 
)
 self.block3 = nn.Sequential(
 SENextBottleneck(encoder_channels[2], encoder_channels[3], stride=stride, is_shortcut=True, pool='max'),
*[SENextBottleneck(encoder_channels[3], encoder_channels[3], stride=1, is_shortcut=False)for i in range(backbone[2])] 
)
 self.block4 = nn.Sequential(
 SENextBottleneck(encoder_channels[3], encoder_channels[4], stride=stride, is_shortcut=True, pool='avg'),
*[SENextBottleneck(encoder_channels[4], encoder_channels[4], stride=1, is_shortcut=False)for i in range(backbone[3])] 
)
 def forward(self, x):
x0 = self.block0(x)
 x1 = self.block1(x0)
 x2 = self.block2(x1)
 x3 = self.block3(x2)
 x4 = self.block4(x3)
 return [x0, x1, x2, x3, x4]
class SCSEModule(nn.Module):
def __init__(self, in_channels, reduction=16):
super().__init__()
self.cSE = nn.Sequential(
 nn.AdaptiveAvgPool1d(1),
nn.Conv1d(in_channels, in_channels // reduction, 1),
nn.ReLU(inplace=True),
nn.Conv1d(in_channels // reduction, in_channels, 1),
nn.Sigmoid() , 
)
 self.sSE = nn.Sequential(nn.Conv1d(in_channels, 1, 1), nn.Sigmoid())
 def forward(self, x):
return x * self.cSE(x)+ x * self.sSE(x)
 class DecoderBlock(nn.Module):
def __init__(self, in_channels, skip_channels, out_channels):
super().__init__()
self.conv1 = ConvBR1d(in_channels + skip_channels, out_channels, kernel_size=3, padding=1)
 self.conv2 = ConvBR1d(out_channels, out_channels, kernel_size=3, padding=1)
 def forward(self, x, skip=None):
x = F.interpolate(x, scale_factor=stride, mode="linear", align_corners=True)
 if skip is not None:
x = torch.cat([x, skip], dim=1)
 x = self.conv1(x)
 x = self.conv2(x)
 return x
class Decoder(nn.Module):
def __init__(self):
super().__init__()
self.block4 = DecoderBlock(encoder_channels[-1], encoder_channels[-2], decoder_channels[0])
 self.block3 = DecoderBlock(decoder_channels[0], encoder_channels[-3], decoder_channels[1])
 self.block2 = DecoderBlock(decoder_channels[1], encoder_channels[-4], decoder_channels[2])
 self.block1 = DecoderBlock(decoder_channels[2], encoder_channels[-5], decoder_channels[3])
 def forward(self, xs):
x = self.block4(xs[4], xs[3])
 x = self.block3(x, xs[2])
 x = self.block2(x, xs[1])
 x = self.block1(x, xs[0])
 return x
class Unet(nn.Module):
def __init__(self, num_features=1, num_classes=11):
super().__init__()
self.num_features = num_features
self.num_classes = num_classes
self.encoder = Encoder(num_features=num_features)
 self.decoder = Decoder()
self.segmentation_head = nn.Conv1d(decoder_channels[-1], num_classes, kernel_size=1, padding=0, stride=1)
 def forward(self, x):
features = self.encoder(x)
 x = self.decoder(features)
 x = self.segmentation_head(x)
 return x
coefficients_ = coefficients.copy()
coefficients_[0] = 1.66
coefficients_[1] = 2.13
coefficients_[3] = 2.85
train_predictions = optR.predict(oof_train, coefficients_ ).astype(np.int8)
 print(f'train pred distribution: {Counter(train_predictions)}')
 test_predictions = optR.predict(oof_test.mean(axis=1), coefficients_ ).astype(np.int8)
 print(f'test pred distribution: {Counter(test_predictions)}' )<init_hyperparams>
class ClassificationMeter:

 def __init__(self, nCls, eps=1e-5, names=None):
self.nCls = nCls
self.names = names
self.eps = eps
self.N = 0
self.table = np.zeros(( self.nCls, 4), dtype=np.int32)
 self._measure = None
def clear(self):
self.N = 0
self.table = np.zeros(( self.nCls, 4), dtype=np.int32)
 def prepare_inputs(self, outputs, truths):

 if torch.is_tensor(outputs):
outputs = outputs.detach().cpu().numpy()
if torch.is_tensor(truths):
truths = truths.detach().cpu().numpy()
return outputs, truths
def update(self, preds, truths):
preds, truths = self.prepare_inputs(preds, truths)
 self.N += np.prod(truths.shape)
 for Cls in range(self.nCls):
true_positive = np.count_nonzero(np.bitwise_and(preds == Cls, truths == Cls))
true_negative = np.count_nonzero(np.bitwise_and(preds != Cls, truths != Cls))
false_positive = np.count_nonzero(np.bitwise_and(preds == Cls, truths != Cls))
false_negative = np.count_nonzero(np.bitwise_and(preds != Cls, truths == Cls))
self.table[Cls] += [true_positive, true_negative, false_positive, false_negative]
def measure(self):
return self._measure
def better(self, A, B):
return A > B
def write(self, writer, global_step, prefix=""):
writer.add_scalar(os.path.join(prefix, "Accuracy"), self.measure() , global_step)
 def report(self, each_class=False):
precisions = []
recalls = []
f1s = []
for Cls in range(self.nCls):
recall = self.table[Cls,0] /(self.table[Cls,0] + self.table[Cls,3] + self.eps)
 precision = self.table[Cls,0] /(self.table[Cls,0] + self.table[Cls,2] + self.eps)
 f1 = 2 * precision * recall /(precision + recall + self.eps)
 precisions.append(precision)
 recalls.append(recall)
 f1s.append(f1)
 total_TP = np.sum(self.table[:, 0])
 accuracy = total_TP/self.N
accuracy_mean_class = np.mean(precisions)
 macro_f1 = np.mean(f1s)
 self._measure = macro_f1
text = f"Macro F1 = {macro_f1:.4f}
"
text += f"\tOverall Accuracy = {accuracy:.4f}({total_TP}/{self.N})
 "
text += f"\tMean-class Accuracy = {accuracy_mean_class:.4f}
"
if each_class:
for Cls in range(self.nCls):
text += f"\tClass {str(Cls)+'('+self.names[Cls]+')' if self.names is not None else Cls}: precision = {precisions[Cls]:.3f} recall = {recalls[Cls]:.3f}
"
return text
lgb_params = {'application': 'regression',
'boosting': 'gbdt',
'metric': 'rmse',
'num_leaves': 70,
'max_depth': 9,
'learning_rate': 0.03,
'bagging_fraction': 0.85,
'feature_fraction': 0.8,
'min_split_gain': 0.02,
'min_child_samples': 150,
'min_child_weight': 0.02,
'lambda_l2': 0.0475,
'verbosity': -1,
'data_random_seed': 17}<train_model>
if TRAIN:
batch = 1; a = 500000*(batch-1); b = 500000*batch
batch = 2; c = 500000*(batch-1); d = 500000*batch
X_train_1s = np.concatenate([train.signal.values[a:b],train.signal.values[c:d]] ).reshape(( -1,1))
y_train_1s = np.concatenate([train.open_channels.values[a:b],train.open_channels.values[c:d]] ).reshape(( -1,1))
batch = 3; a = 500000*(batch-1); b = 500000*batch
batch = 7; c = 500000*(batch-1); d = 500000*batch
X_train_1f = np.concatenate([train.signal.values[a:b],train.signal.values[c:d]] ).reshape(( -1,1))
y_train_1f = np.concatenate([train.open_channels.values[a:b],train.open_channels.values[c:d]] ).reshape(( -1,1))
batch = 4; a = 500000*(batch-1); b = 500000*batch
batch = 8; c = 500000*(batch-1); d = 500000*batch
X_train_3 = np.concatenate([train.signal.values[a:b],train.signal.values[c:d]] ).reshape(( -1,1))
y_train_3 = np.concatenate([train.open_channels.values[a:b],train.open_channels.values[c:d]] ).reshape(( -1,1))
batch = 6; a = 500000*(batch-1); b = 500000*batch
batch = 9; c = 500000*(batch-1); d = 500000*batch
X_train_5 = np.concatenate([train.signal.values[a:b],train.signal.values[c:d]] ).reshape(( -1,1))
y_train_5 = np.concatenate([train.open_channels.values[a:b],train.open_channels.values[c:d]] ).reshape(( -1,1))
batch = 5; a = 500000*(batch-1); b = 500000*batch
batch = 10; c = 500000*(batch-1); d = 500000*batch
X_train_10 = np.concatenate([train.signal.values[a:b],train.signal.values[c:d]] ).reshape(( -1,1))
y_train_10 = np.concatenate([train.open_channels.values[a:b],train.open_channels.values[c:d]] ).reshape(( -1,1))
for X, y, num_classes, model_name, max_epoch in zip(
 [X_train_1s, X_train_1f, X_train_3, X_train_5, X_train_10],
[y_train_1s, y_train_1f, y_train_3, y_train_5, y_train_10],
np.array([1, 1, 3, 5, 10])+1,
['1s', '1f', '3', '5', '10'],
[200, 100, 100, 100, 100], 
):
X = X.reshape(-1, 1, time_step)
 y = y.reshape(-1, time_step)
 idx = np.arange(X.shape[0])
 kf = model_selection.KFold(5, shuffle=True, random_state=seed)
 train_idx, val_idx = list(kf.split(idx)) [fold]
print("model_name:", model_name)
 print("train dataset shape:", X[train_idx].shape, y[train_idx].shape)
 train_dataset = IonDataset(X[train_idx], y[train_idx], 'train', train_augment)
 valid_dataset = IonDataset(X[val_idx], y[val_idx], 'train', valid_augment)
 loaders = {
'train': DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=4, pin_memory=True),
'valid': DataLoader(valid_dataset, batch_size=batch_size, shuffle=False, num_workers=4, pin_memory=True),
}
model = Unet(num_classes=num_classes)
 loss_function = nn.CrossEntropyLoss()
optimizer = optim.AdamW(model.parameters() , lr=0.001, weight_decay=1e-5)
 scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, 'min', patience=10, factor=0.1, min_lr=1e-8)
 metrics = [ClassificationMeter(num_classes),]
trainer = Trainer(device, workspace, model, optimizer, scheduler, loss_function, loaders, metrics,
model_name=model_name,
report_step_interval=-1, 
)
 trainer.train(max_epoch )
def lgb_train(params, X_train, X_test):
early_stop = 500
verbose_eval = 1000
num_rounds = 15000
n_splits = 5
kfold = StratifiedKFold(n_splits=n_splits, random_state=1337)
 oof_train = np.zeros(( X_train.shape[0]))
oof_test = np.zeros(( X_test.shape[0], n_splits))
i = 0
for train_index, valid_index in kfold.split(X_train, X_train['AdoptionSpeed'].values):
X_tr = X_train.iloc[train_index, :]
X_val = X_train.iloc[valid_index, :]
y_tr = X_tr['AdoptionSpeed'].values
X_tr = X_tr.drop(['AdoptionSpeed'], axis=1)
 y_val = X_val['AdoptionSpeed'].values
X_val = X_val.drop(['AdoptionSpeed'], axis=1)
 print('
y_tr distribution: {}'.format(Counter(y_tr)))
 d_train = lgb.Dataset(X_tr, label=y_tr)
 d_valid = lgb.Dataset(X_val, label=y_val)
 watchlist = [d_train, d_valid]
print('training LGB:')
 model = lgb.train(params,
train_set=d_train,
num_boost_round=num_rounds,
valid_sets=watchlist,
verbose_eval=verbose_eval,
early_stopping_rounds=early_stop)
 val_pred = model.predict(X_val, num_iteration=model.best_iteration)
 test_pred = model.predict(X_test, num_iteration=model.best_iteration)
 oof_train[valid_index] = val_pred
oof_test[:, i] = test_pred
i += 1
return oof_train, oof_test<train_model>
if PREDICT:
sub = pd.read_csv(".. /input/liverpool-ion-switching/sample_submission.csv", dtype={'time':str})
 predictors = {}
for num_classes, model_name in zip(
 np.array([1, 1, 3, 5, 10])+1,
['1s', '1f', '3', '5', '10'] 
):
if not ENSEMBLE:
models = [Unet(num_classes=num_classes),]
predictor = Predictor(device, models)
 predictor.load_checkpoint(os.path.join(workspace, f"{model_name}_best.pth.tar"), 0)
 else:
models = [
Unet(num_classes=num_classes),
]
predictor = Predictor(device, models)
 predictor.load_checkpoint(os.path.join("workspace/unet-wide-4k-f0", f"{model_name}_best.pth.tar"), 0)
 predictor.load_checkpoint(os.path.join("workspace/unet-wide-4k-f1", f"{model_name}_best.pth.tar"), 1)
 predictors[model_name] = predictor
for start, end, model_type in zip(
 np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10])*100000,
np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 20])*100000,
['1s', '3', '5', '1s', '1f', '10', '5', '10', '1s', '3', '1s'], 
):
X = test.signal.values[start:end].reshape(-1, 1, time_step_test)
 test_dataset = IonDataset(X, type='test')
 test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)
 res = predictors[model_type].predict(test_loader)
 sub.iloc[start:end,1] = res
sub.to_csv("pred.csv", index=False)
 plt.figure(figsize=(20,5))
res = 1000
let = ['A','B','C','D','E','F','G','H','I','J']
plt.plot(range(0,test.shape[0],res),sub.open_channels[0::res])
 for i in range(5): plt.plot([i*500000,i*500000],[-5,12.5],'r')
 for i in range(21): plt.plot([i*100000,i*100000],[-5,12.5],'r:')
 for k in range(4): plt.text(k*500000+250000,10,str(k+1),size=20)
 for k in range(10): plt.text(k*100000+40000,7.5,let[k],size=16)
 plt.title('Test Data Predictions',size=16)
 plt.show()