Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<categorify>
train = pd.read_csv('.. /input/train.csv')
 train['SalePrice'] = np.log1p(train['SalePrice'])
 test = pd.read_csv('.. /input/test.csv')
 sub = pd.read_csv('.. /input/sample_submission.csv')
 print(train.shape, test.shape )
data_type = 'test'
test_preds_list = []
for i in range(4):
gc.collect()
indices = [i]
test_images = prepare_image(data_type,indices=indices)
 n_dataset = len(test_images)
 print(f'i={i}, n_dataset={n_dataset}')
 test_dataset = BengaliAIDataset(test_images)
 print('test_dataset', len(test_dataset))
test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=16, shuffle=False)
 for idx, inputs in enumerate(test_loader):
inputs = inputs.to(device)
 output = model(inputs.float())
 outputs = torch.split(output, [168, 11, 7], dim=1)
 outputs = outputs[0].argmax(1), outputs[1].argmax(1), outputs[2].argmax(1)
 test_preds_list.append(outputs)
 del test_images
gc.collect()<categorify>
col = [c for c in train.columns if c not in ['Id', 'SalePrice']]
train['null_count'] = np.sum(( train[col]==np.nan ).values, axis=1)
 test['null_count'] = np.sum(( test[col]==np.nan ).values, axis=1)
 for c in col:
train[c + '_null'] =(train[c]==np.nan ).astype(int)
 test[c + '_null'] =(test[c]==np.nan ).astype(int)
 if train[c].dtype == 'O':
lbl0 = {k: v for k, v in train.groupby(by=[c], as_index=False)['SalePrice'].mean().values}
lbl1 = {k: v for k, v in train.groupby(by=[c], as_index=False)['SalePrice'].max().values}
lbl2 = {k: v for k, v in train.groupby(by=[c], as_index=False)['SalePrice'].min().values}
lbl3 = {k:v for v, k in enumerate(train[c].value_counts().index)}
lbl4 = {k: np.log1p(v)for k, v in train[c].value_counts().reset_index().values}
lbl5 = {k:v for v, k in enumerate(pd.concat(( train[c], test[c])).value_counts().index)}
lbl6 = {k: np.log1p(v)for k, v in pd.concat(( train[c], test[c])).value_counts().reset_index().values}
if len(lbl1)>100:
for k in lbl1:
train[c+'_'+str(k)] =(train[c] == k ).astype(int)
 test[c+'_'+str(k)] =(test[c] == k ).astype(int)
 for i in range(6):
train[c+'_key_'+str(i)] = train[c].map(eval('lbl'+str(i)))
 test[c+'_key_'+str(i)] = test[c].map(eval('lbl'+str(i)))
 train[c] = train[c].map(lbl6)
 test[c] = test[c].map(lbl6)
 else:
slope, intercept, r_value, p_value, std_err = stats.linregress(train[c],train['SalePrice'])
 train[c+'_slope'] = train[c] * slope
test[c+'_slope'] = test[c] * slope
train[c+'_log1p'] = np.log1p(train[c])
 test[c+'_log1p'] = np.log1p(test[c])
 train.fillna(-1, inplace=True)
 test.fillna(-1, inplace=True)
 feature_cnt = 25
col = [c for c in train.columns if c not in ['Id', 'SalePrice']]
dim_reduction_pca = decomposition.PCA(n_components=feature_cnt, random_state=10)
 dim_reduction_svd = decomposition.TruncatedSVD(n_components=feature_cnt, random_state=11)
 dim_reduction_pca_train = dim_reduction_pca.fit_transform(train[col])
 dim_reduction_svd_train = dim_reduction_svd.fit_transform(train[col])
 dim_reduction_pca_test = dim_reduction_pca.transform(test[col])
 dim_reduction_svd_test = dim_reduction_svd.transform(test[col])
 for i in range(feature_cnt):
train['dim_reduction_pca_'+str(i)] = dim_reduction_pca_train[:,i]
test['dim_reduction_pca_'+str(i)] = dim_reduction_pca_test[:,i]
train['dim_reduction_svd_'+str(i)] = dim_reduction_svd_train[:,i]
test['dim_reduction_svd_'+str(i)] = dim_reduction_svd_test[:,i]
print(train.shape, test.shape )
p0 = np.concatenate([test_preds[0].cpu().detach().numpy() for test_preds in test_preds_list], axis=0)
 p1 = np.concatenate([test_preds[1].cpu().detach().numpy() for test_preds in test_preds_list], axis=0)
 p2 = np.concatenate([test_preds[2].cpu().detach().numpy() for test_preds in test_preds_list], axis=0)
 print('p0', p0.shape, 'p1', p1.shape, 'p2', p2.shape )<define_variables>
col = [c for c in train.columns if c not in ['Id', 'SalePrice']]
col = [c for c in col if train[c].dtype != 'O']
x1, x2, y1, y2 = train_test_split(train[col].fillna(-1), train['SalePrice'], test_size=0.2, random_state=3)
 etr = ExtraTreesRegressor(n_jobs=-1, random_state=3)
 etr.fit(x1, y1)
 print(np.sqrt(metrics.mean_squared_error(y2, etr.predict(x2))))
feature_importances = pd.DataFrame({'features':col, 'importance': etr.feature_importances_} ).sort_values(by=['importance'], ascending=False)
 feature_importances.head()
row_id = []
target = []
for i in range(len(p0)) :
row_id += [f'Test_{i}_consonant_diacritic', f'Test_{i}_grapheme_root',
f'Test_{i}_vowel_diacritic']
target += [p2[i], p0[i], p1[i]]<save_to_csv>
d_imp = {k:v for k, v in feature_importances.values}
def f_hl_test(a):
q1 = a.quantile(0.25)
 q3 = a.quantile(0.75)
 iqr = q3 - q1
iqr_h = q3 + iqr*1.5
iqr_l = q1 - iqr*1.5
return iqr_l, iqr_h
train['outlier'] = 0.0
test['outlier'] = 0.0
for c in feature_importances.features:
l, h = f_hl_test(train[c])
 train['outlier'] +=(((train[c] < l ).astype(int)+(train[c] > h ).astype(int)) > 0 ).astype(int)* d_imp[c]
test['outlier'] +=(((test[c] < l ).astype(int)+(test[c] > h ).astype(int)) > 0 ).astype(int)* d_imp[c]
print(train.shape, test.shape )
submission_df = pd.DataFrame({'row_id': row_id, 'target': target})
 submission_df.to_csv('submission.csv', index=False )<set_options>
sk_if = IsolationForest(n_jobs=-1, random_state=3)
 sk_lof = LocalOutlierFactor(novelty=True, n_jobs=-1)
 train['outlier_sk_if'] = sk_if.fit_predict(train[col].fillna(-1), train['SalePrice'])
 sk_lof.fit(train[col].fillna(-1), train['SalePrice'])
 train['outlier_sk_lof'] = sk_lof.predict(train[col].fillna(-1))
test['outlier_sk_if'] = sk_if.predict(test[col].fillna(-1))
test['outlier_sk_lof'] = sk_lof.predict(test[col].fillna(-1))
%%writefile /opt/conda/lib/python3.6/site-packages/tensorflow_core/python/keras/saving/hdf5_format.py

 try:
HDF5_OBJECT_HEADER_LIMIT = 64512
except ImportError:
h5py = None
def save_model_to_hdf5(model, filepath, overwrite=True, include_optimizer=True):

 if h5py is None:
raise ImportError('`save_model` requires h5py.')
 if len(model.weights)!= len(model._undeduplicated_weights):
logging.warning('Found duplicated `Variable`s in Model's `weights`.'
'This is usually caused by `Variable`s being shared by '
'Layers in the Model.These `Variable`s will be treated '
'as separate `Variable`s when the Model is restored.To '
'avoid this, please save with `save_format="tf"`.')
 if not isinstance(filepath, h5py.File):
if not overwrite and os.path.isfile(filepath):
proceed = ask_to_proceed_with_overwrite(filepath)
 if not proceed:
return
f = h5py.File(filepath, mode='w')
 opened_new_file = True
else:
f = filepath
opened_new_file = False
try:
model_metadata = saving_utils.model_metadata(model, include_optimizer)
 for k, v in model_metadata.items() :
if isinstance(v,(dict, list, tuple)) :
f.attrs[k] = json.dumps(
 v, default=serialization.get_json_type ).encode('utf8')
 else:
f.attrs[k] = v
model_weights_group = f.create_group('model_weights')
 model_layers = model.layers
save_weights_to_hdf5_group(model_weights_group, model_layers)
 if(include_optimizer and model.optimizer and
not isinstance(model.optimizer, optimizers.TFOptimizer)) :
save_optimizer_weights_to_hdf5_group(f, model.optimizer)
 f.flush()
finally:
if opened_new_file:
f.close()
def load_model_from_hdf5(filepath, custom_objects=None, compile=True):

 if h5py is None:
raise ImportError('`load_model` requires h5py.')
 if not custom_objects:
custom_objects = {}
opened_new_file = not isinstance(filepath, h5py.File)
 if opened_new_file:
f = h5py.File(filepath, mode='r')
 else:
f = filepath
model = None
try:
model_config = f.attrs.get('model_config')
 if model_config is None:
raise ValueError('No model found in config file.')
 model_config = json.loads(model_config.decode('utf-8'))
model = model_config_lib.model_from_config(model_config,
custom_objects=custom_objects)
 load_weights_from_hdf5_group(f['model_weights'], model.layers)
 if compile:
training_config = f.attrs.get('training_config')
 if training_config is None:
logging.warning('No training configuration found in the save file, so '
'the model was *not* compiled.Compile it manually.')
 return model
training_config = json.loads(training_config.decode('utf-8'))
model.compile(**saving_utils.compile_args_from_training_config(
 training_config, custom_objects))
if 'optimizer_weights' in f:
if model._is_graph_network:
if not ops.executing_eagerly_outside_functions() :
model._make_train_function()
optimizer_weight_values = load_optimizer_weights_from_hdf5_group(f)
 try:
model.optimizer.set_weights(optimizer_weight_values)
 except ValueError:
logging.warning('Error in loading the saved optimizer '
'state.As a result, your model is '
'starting with a freshly initialized '
'optimizer.')
 else:
logging.warning('Sequential models without an `input_shape` '
'passed to the first layer cannot reload their '
'optimizer state.As a result, your model is'
'starting with a freshly initialized optimizer.')
 finally:
if opened_new_file:
f.close()
return model
def preprocess_weights_for_loading(layer,
weights,
original_keras_version=None,
original_backend=None):

 def convert_nested_bidirectional(weights):

 num_weights_per_layer = len(weights)// 2
forward_weights = preprocess_weights_for_loading(
 layer.forward_layer, weights[:num_weights_per_layer],
original_keras_version, original_backend)
 backward_weights = preprocess_weights_for_loading(
 layer.backward_layer, weights[num_weights_per_layer:],
original_keras_version, original_backend)
 return forward_weights + backward_weights
def convert_nested_time_distributed(weights):

 return preprocess_weights_for_loading(
 layer.layer, weights, original_keras_version, original_backend)
 def convert_nested_model(weights):

 trainable_weights = weights[:len(layer.trainable_weights)]
non_trainable_weights = weights[len(layer.trainable_weights):]
new_trainable_weights = []
new_non_trainable_weights = []
for sublayer in layer.layers:
num_trainable_weights = len(sublayer.trainable_weights)
 num_non_trainable_weights = len(sublayer.non_trainable_weights)
 if sublayer.weights:
preprocessed = preprocess_weights_for_loading(
 layer=sublayer,
weights=(trainable_weights[:num_trainable_weights] +
non_trainable_weights[:num_non_trainable_weights]),
original_keras_version=original_keras_version,
original_backend=original_backend)
 new_trainable_weights.extend(preprocessed[:num_trainable_weights])
 new_non_trainable_weights.extend(preprocessed[num_trainable_weights:])
 trainable_weights = trainable_weights[num_trainable_weights:]
non_trainable_weights = non_trainable_weights[
num_non_trainable_weights:]
return new_trainable_weights + new_non_trainable_weights
if layer.__class__.__name__ == 'Bidirectional':
weights = convert_nested_bidirectional(weights)
 if layer.__class__.__name__ == 'TimeDistributed':
weights = convert_nested_time_distributed(weights)
 elif layer.__class__.__name__ in ['Model', 'Sequential']:
weights = convert_nested_model(weights)
 if original_keras_version == '1':
if layer.__class__.__name__ == 'TimeDistributed':
weights = preprocess_weights_for_loading(
 layer.layer, weights, original_keras_version, original_backend)
 if layer.__class__.__name__ == 'Conv1D':
shape = weights[0].shape
if shape[:2] !=(layer.kernel_size[0], 1)or shape[3] != layer.filters:
assert shape[0] == layer.filters and shape[2:] ==(layer.kernel_size[0],
1)
 weights[0] = np.transpose(weights[0],(2, 3, 1, 0))
weights[0] = weights[0][:, 0, :, :]
if layer.__class__.__name__ == 'Conv2D':
if layer.data_format == 'channels_first':
weights[0] = np.transpose(weights[0],(2, 3, 1, 0))
if layer.__class__.__name__ == 'Conv2DTranspose':
if layer.data_format == 'channels_last':
weights[0] = np.transpose(weights[0],(0, 1, 3, 2))
if layer.data_format == 'channels_first':
weights[0] = np.transpose(weights[0],(2, 3, 0, 1))
if layer.__class__.__name__ == 'Conv3D':
if layer.data_format == 'channels_first':
weights[0] = np.transpose(weights[0],(2, 3, 4, 1, 0))
if layer.__class__.__name__ == 'GRU':
if len(weights)== 9:
kernel = np.concatenate([weights[0], weights[3], weights[6]], axis=-1)
 recurrent_kernel = np.concatenate(
 [weights[1], weights[4], weights[7]], axis=-1)
 bias = np.concatenate([weights[2], weights[5], weights[8]], axis=-1)
 weights = [kernel, recurrent_kernel, bias]
if layer.__class__.__name__ == 'LSTM':
if len(weights)== 12:
kernel = np.concatenate(
 [weights[0], weights[6], weights[3], weights[9]], axis=-1)
 recurrent_kernel = np.concatenate(
 [weights[1], weights[7], weights[4], weights[10]], axis=-1)
 bias = np.concatenate(
 [weights[2], weights[8], weights[5], weights[11]], axis=-1)
 weights = [kernel, recurrent_kernel, bias]
if layer.__class__.__name__ == 'ConvLSTM2D':
if len(weights)== 12:
kernel = np.concatenate(
 [weights[0], weights[6], weights[3], weights[9]], axis=-1)
 recurrent_kernel = np.concatenate(
 [weights[1], weights[7], weights[4], weights[10]], axis=-1)
 bias = np.concatenate(
 [weights[2], weights[8], weights[5], weights[11]], axis=-1)
 if layer.data_format == 'channels_first':
kernel = np.transpose(kernel,(2, 3, 1, 0))
recurrent_kernel = np.transpose(recurrent_kernel,(2, 3, 1, 0))
weights = [kernel, recurrent_kernel, bias]
conv_layers = ['Conv1D', 'Conv2D', 'Conv3D', 'Conv2DTranspose', 'ConvLSTM2D']
if layer.__class__.__name__ in conv_layers:
if original_backend == 'theano':
weights[0] = conv_utils.convert_kernel(weights[0])
 if layer.__class__.__name__ == 'ConvLSTM2D':
weights[1] = conv_utils.convert_kernel(weights[1])
 if K.int_shape(layer.weights[0])!= weights[0].shape:
weights[0] = np.transpose(weights[0],(3, 2, 0, 1))
if layer.__class__.__name__ == 'ConvLSTM2D':
weights[1] = np.transpose(weights[1],(3, 2, 0, 1))
return _convert_rnn_weights(layer, weights)
 def _convert_rnn_weights(layer, weights):

 def transform_kernels(kernels, func, n_gates):

 return np.hstack([func(k)for k in np.hsplit(kernels, n_gates)])
 def transpose_input(from_cudnn):

 order = 'F' if from_cudnn else 'C'
def transform(kernel):
return kernel.T.reshape(kernel.shape, order=order)
 return transform
target_class = layer.__class__.__name__
if target_class in ['LSTM', 'CuDNNLSTM'] and len(weights)== 3:
units = weights[1].shape[0]
bias_shape = weights[2].shape
n_gates = 4
if bias_shape ==(2 * units * n_gates,):
source = 'CuDNNLSTM'
elif bias_shape ==(units * n_gates,):
source = 'LSTM'
else:
raise ValueError('Invalid bias shape: ' + str(bias_shape))
def convert_lstm_weights(weights, from_cudnn=True):

 kernels = transform_kernels(weights[0], transpose_input(from_cudnn),
n_gates)
 recurrent_kernels = transform_kernels(weights[1], lambda k: k.T, n_gates)
 if from_cudnn:
biases = np.sum(np.split(weights[2], 2, axis=0), axis=0)
 else:
biases = np.tile(0.5 * weights[2], 2)
 return [kernels, recurrent_kernels, biases]
if source != target_class:
weights = convert_lstm_weights(weights, from_cudnn=source == 'CuDNNLSTM')
 if target_class in ['GRU', 'CuDNNGRU'] and len(weights)== 3:
units = weights[1].shape[0]
bias_shape = weights[2].shape
n_gates = 3
def convert_gru_weights(weights, from_cudnn=True):

 kernels = transform_kernels(weights[0], transpose_input(from_cudnn),
n_gates)
 recurrent_kernels = transform_kernels(weights[1], lambda k: k.T, n_gates)
 biases = np.array(weights[2] ).reshape(( 2, -1)if from_cudnn else -1)
 return [kernels, recurrent_kernels, biases]
if bias_shape ==(2 * units * n_gates,):
source = 'CuDNNGRU'
elif bias_shape ==(2, units * n_gates):
source = 'GRU(reset_after=True)'
elif bias_shape ==(units * n_gates,):
source = 'GRU(reset_after=False)'
else:
raise ValueError('Invalid bias shape: ' + str(bias_shape))
if target_class == 'CuDNNGRU':
target = 'CuDNNGRU'
elif layer.reset_after:
target = 'GRU(reset_after=True)'
else:
target = 'GRU(reset_after=False)'
if source != target:
types =(source, target)
 if 'GRU(reset_after=False)' in types:
raise ValueError('%s is not compatible with %s' % types)
 if source == 'CuDNNGRU':
weights = convert_gru_weights(weights, from_cudnn=True)
 elif source == 'GRU(reset_after=True)':
weights = convert_gru_weights(weights, from_cudnn=False)
 return weights
def save_optimizer_weights_to_hdf5_group(hdf5_group, optimizer):

 symbolic_weights = getattr(optimizer, 'weights')
 if symbolic_weights:
weights_group = hdf5_group.create_group('optimizer_weights')
 weight_names = [str(w.name ).encode('utf8')for w in symbolic_weights]
save_attributes_to_hdf5_group(weights_group, 'weight_names', weight_names)
 weight_values = K.batch_get_value(symbolic_weights)
 for name, val in zip(weight_names, weight_values):
param_dset = weights_group.create_dataset(
 name, val.shape, dtype=val.dtype)
 if not val.shape:
param_dset[() ] = val
else:
param_dset[:] = val
def load_optimizer_weights_from_hdf5_group(hdf5_group):

 weights_group = hdf5_group['optimizer_weights']
optimizer_weight_names = load_attributes_from_hdf5_group(
 weights_group, 'weight_names')
 return [weights_group[weight_name] for weight_name in optimizer_weight_names]
def save_weights_to_hdf5_group(f, layers):

 save_attributes_to_hdf5_group(
 f, 'layer_names', [layer.name.encode('utf8')for layer in layers])
 f.attrs['backend'] = K.backend().encode('utf8')
 f.attrs['keras_version'] = str(keras_version ).encode('utf8')
 for layer in sorted(layers, key=lambda x: x.name):
g = f.create_group(layer.name)
 weights = _legacy_weights(layer)
 weight_values = K.batch_get_value(weights)
 weight_names = [w.name.encode('utf8')for w in weights]
save_attributes_to_hdf5_group(g, 'weight_names', weight_names)
 for name, val in zip(weight_names, weight_values):
param_dset = g.create_dataset(name, val.shape, dtype=val.dtype)
 if not val.shape:
param_dset[() ] = val
else:
param_dset[:] = val
def load_weights_from_hdf5_group(f, layers):

 if 'keras_version' in f.attrs:
original_keras_version = f.attrs['keras_version'].decode('utf8')
 else:
original_keras_version = '1'
if 'backend' in f.attrs:
original_backend = f.attrs['backend'].decode('utf8')
 else:
original_backend = None
filtered_layers = []
for layer in layers:
weights = _legacy_weights(layer)
 if weights:
filtered_layers.append(layer)
 layer_names = load_attributes_from_hdf5_group(f, 'layer_names')
 filtered_layer_names = []
for name in layer_names:
g = f[name]
weight_names = load_attributes_from_hdf5_group(g, 'weight_names')
 if weight_names:
filtered_layer_names.append(name)
 layer_names = filtered_layer_names
if len(layer_names)!= len(filtered_layers):
raise ValueError('You are trying to load a weight file '
'containing ' + str(len(layer_names)) +
' layers into a model with ' + str(len(filtered_layers)) +
' layers.')
 weight_value_tuples = []
for k, name in enumerate(layer_names):
g = f[name]
weight_names = load_attributes_from_hdf5_group(g, 'weight_names')
 weight_values = [np.asarray(g[weight_name])for weight_name in weight_names]
layer = filtered_layers[k]
symbolic_weights = _legacy_weights(layer)
 weight_values = preprocess_weights_for_loading(
 layer, weight_values, original_keras_version, original_backend)
 if len(weight_values)!= len(symbolic_weights):
raise ValueError('Layer
'" in the current model)was found to '
'correspond to layer ' + name + ' in the save file.'
'However the new layer ' + layer.name + ' expects ' +
str(len(symbolic_weights)) +
' weights, but the saved weights have ' +
str(len(weight_values)) + ' elements.')
 weight_value_tuples += zip(symbolic_weights, weight_values)
 K.batch_set_value(weight_value_tuples)
 def load_weights_from_hdf5_group_by_name(
 f, layers, skip_mismatch=False):

 if 'keras_version' in f.attrs:
original_keras_version = f.attrs['keras_version'].decode('utf8')
 else:
original_keras_version = '1'
if 'backend' in f.attrs:
original_backend = f.attrs['backend'].decode('utf8')
 else:
original_backend = None
layer_names = load_attributes_from_hdf5_group(f, 'layer_names')
 index = {}
for layer in layers:
if layer.name:
index.setdefault(layer.name, [] ).append(layer)
 weight_value_tuples = []
for k, name in enumerate(layer_names):
g = f[name]
weight_names = load_attributes_from_hdf5_group(g, 'weight_names')
 weight_values = [np.asarray(g[weight_name])for weight_name in weight_names]
for layer in index.get(name, []):
symbolic_weights = _legacy_weights(layer)
 weight_values = preprocess_weights_for_loading(
 layer, weight_values, original_keras_version, original_backend)
 if len(weight_values)!= len(symbolic_weights):
if skip_mismatch:
logging.warning('Skipping loading of weights for '
'layer {}'.format(layer.name)+ ' due to mismatch '
'in number of weights({} vs {} ).'.format(
 len(symbolic_weights), len(weight_values)))
 continue
raise ValueError('Layer
'")expects ' + str(len(symbolic_weights)) +
' weight(s), but the saved weights' + ' have ' +
str(len(weight_values)) + ' element(s ).')
 for i in range(len(weight_values)) :
if K.int_shape(symbolic_weights[i])!= weight_values[i].shape:
if skip_mismatch:
logging.warning('Skipping loading of weights for '
'layer {}'.format(layer.name)+ ' due to '
'mismatch in shape({} vs {} ).'.format(
 symbolic_weights[i].shape,
weight_values[i].shape))
continue
raise ValueError('Layer
'"), weight ' + str(symbolic_weights[i])+
' has shape {}'.format(K.int_shape(
 symbolic_weights[i])) +
', but the saved weight has shape ' +
str(weight_values[i].shape)+ '.')
 else:
weight_value_tuples.append(( symbolic_weights[i], weight_values[i]))
K.batch_set_value(weight_value_tuples)
 def save_attributes_to_hdf5_group(group, name, data):

 bad_attributes = [x for x in data if len(x)> HDF5_OBJECT_HEADER_LIMIT]
if bad_attributes:
raise RuntimeError('The following attributes cannot be saved to HDF5 '
'file because they are larger than %d bytes: %s' % 
(HDF5_OBJECT_HEADER_LIMIT, ', '.join(bad_attributes)))
 data_npy = np.asarray(data)
 num_chunks = 1
chunked_data = np.array_split(data_npy, num_chunks)
 while any(x.nbytes > HDF5_OBJECT_HEADER_LIMIT for x in chunked_data):
num_chunks += 1
chunked_data = np.array_split(data_npy, num_chunks)
 if num_chunks > 1:
for chunk_id, chunk_data in enumerate(chunked_data):
group.attrs['%s%d' %(name, chunk_id)] = chunk_data
else:
group.attrs[name] = data
def load_attributes_from_hdf5_group(group, name):

 if name in group.attrs:
data = [n.decode('utf8')for n in group.attrs[name]]
else:
data = []
chunk_id = 0
while '%s%d' %(name, chunk_id)in group.attrs:
data.extend(
 [n.decode('utf8')for n in group.attrs['%s%d' %(name, chunk_id)]])
 chunk_id += 1
return data
def _legacy_weights(layer):

 weights = layer.trainable_weights + layer.non_trainable_weights
if any([not isinstance(w, variables_module.Variable)for w in weights]):
raise NotImplementedError(
 'Save or restore weights that is not an instance of `tf.Variable` is '
'not supported in h5, use `save_format='tf'` instead.Got a model '
'or layer {} with weights {}'.format(layer.__class__.__name__, weights))
return weights<install_modules>
col = [c for c in train.columns if c not in ['Id', 'SalePrice']]
data = simple.Data(train, test, 'Id', 'SalePrice')
 params = {'learning_rate': 0.005, 'max_depth': -1, 'boosting': 'gbdt', 'objective': 'regression', 'metric':'rmse', 'seed': 3, 'num_iterations': 5000, 'early_stopping_round': 200, 'verbose_eval': 300, 'num_leaves': 64, 'feature_fraction': 0.8, 'bagging_fraction': 0.8, 'bagging_freq': 5}
sub = simple.Model(data, 'LGB', params, 0.2, 7 ).PRED
sub['SalePrice'] = np.expm1(sub['SalePrice'])
 sub.to_csv('submission_lgb.csv', index=False )
!pip install.. /input/kaggle-efficientnet-repo/efficientnet-1.0.0-py3-none-any.whl
EarlyStopping, ReduceLROnPlateau, TensorBoard, CSVLogger)
<categorify>
lr = linear_model.Ridge(random_state=3)
 lr.fit(train[col], train['SalePrice'])
 print('Ridge', np.sqrt(metrics.mean_squared_error(train['SalePrice'], lr.predict(train[col]))))
test['SalePrice'] = np.expm1(lr.predict(test[col]))
test[['Id','SalePrice']].to_csv('submission_ridge.csv', index=False)
 sub = simple.Blend(['submission_lgb.csv', 'submission_ridge.csv'], 'Id', 'SalePrice' ).BLEND.reset_index()
def normalize(image):
image -= tf.constant([0.485 * 255, 0.456 * 255, 0.406 * 255])
 image /= tf.constant([0.229 * 255, 0.224 * 255, 0.225 * 255])
 return image
def one_hot(image, label):
label['root'] = tf.one_hot(label['root'], 168)
 label['vowel'] = tf.one_hot(label['vowel'], 11)
 label['consonant'] = tf.one_hot(label['consonant'], 7)
 label['unique'] = tf.one_hot(label['unique'], 1292)
 label['root2'] = tf.one_hot(label['root2'], 168)
 label['vowel2'] = tf.one_hot(label['vowel2'], 11)
 label['consonant2'] = tf.one_hot(label['consonant2'], 7)
 label['unique2'] = tf.one_hot(label['unique2'], 1292)
 label['root3'] = tf.one_hot(label['root3'], 168)
 label['vowel3'] = tf.one_hot(label['vowel3'], 11)
 label['consonant3'] = tf.one_hot(label['consonant3'], 7)
 label['unique3'] = tf.one_hot(label['unique3'], 1292)
 return image, label
def prepare_metric_learning(image, label, mode='train'):
if model == 'train':
return(image, label['root'], label['vowel'], label['consonant'], label['unique']), label
else:
return(image, tf.zeros_like(label['root']), tf.zeros_like(label['vowel']), tf.zeros_like(label['consonant']), tf.zeros_like(label['unique'])) , label
def get_callbacks(work_dir):
checkpoint = ModelCheckpoint(work_dir + '/best.h5',
monitor = 'val_loss',
verbose = 0, save_best_only=True,
mode = 'min',
save_weights_only = True)
 csv_logger = CSVLogger(work_dir + '/log.csv')
 early = EarlyStopping(monitor='val_loss', mode='min', patience=20)
 scheduler = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=10)
 return [checkpoint, csv_logger, early, scheduler]
def read_tfrecords(example, input_size):
features = {
'img': tf.io.FixedLenFeature([], tf.string),
'image_id': tf.io.FixedLenFeature([], tf.int64),
'grapheme_root': tf.io.FixedLenFeature([], tf.int64),
'vowel_diacritic': tf.io.FixedLenFeature([], tf.int64),
'consonant_diacritic': tf.io.FixedLenFeature([], tf.int64),
'unique_tuple': tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, features)
 img = tf.image.decode_image(example['img'])
 img = tf.reshape(img, input_size +(1,))
img = tf.cast(img, tf.float32)
 img = tf.repeat(img, 3, -1)
 grapheme_root = tf.cast(example['grapheme_root'], tf.int32)
 vowel_diacritic = tf.cast(example['vowel_diacritic'], tf.int32)
 consonant_diacritic = tf.cast(example['consonant_diacritic'], tf.int32)
 unique_tuple = tf.cast(example['unique_tuple'], tf.int32)
 return img, {'root': grapheme_root, 'vowel': vowel_diacritic, 'consonant': consonant_diacritic, 'unique': unique_tuple, 'root2': grapheme_root, 'vowel2': vowel_diacritic, 'consonant2': consonant_diacritic, 'unique2': unique_tuple, 'root3': grapheme_root, 'vowel3': vowel_diacritic, 'consonant3': consonant_diacritic, 'unique3': unique_tuple} 
<normalization>
xtrain = train.copy()
for i in range(3):
test.drop(columns=['SalePrice'], inplace=True)
 sub['SalePrice'] = np.log1p(sub['SalePrice'])
 test = pd.merge(test, sub[['Id','SalePrice']], how='left', on='Id')
 train = pd.concat(( xtrain, test), ignore_index=True, sort=False)
 for c in feature_importances.features[:5]:
l, h = f_hl_test(train[c])
 print(c, len(train[(train[c] < l)|(train[c] > h)]))
train = train[(train[c] >= l)&(train[c] <= h)].reset_index(drop=True)
 data = simple.Data(train, test, 'Id', 'SalePrice')
 params = {'learning_rate': 0.02, 'max_depth': -1, 'boosting': 'gbdt', 'objective': 'regression', 'metric':'rmse', 'seed': 3, 'num_iterations': 1000, 'early_stopping_round': 200, 'verbose_eval': 300, 'num_leaves': 64, 'feature_fraction': 0.8, 'bagging_fraction': 0.8, 'bagging_freq': 5}
sub = simple.Model(data, 'LGB', params, 0.2, 7 ).PRED
sub['SalePrice'] = np.expm1(sub['SalePrice'])
 sub.to_csv('submission_lgb.csv', index=False)
 lr.fit(train[col], train['SalePrice'])
 print('Ridge', np.sqrt(metrics.mean_squared_error(train['SalePrice'], lr.predict(train[col]))))
test['SalePrice'] = np.expm1(lr.predict(test[col]))
test[['Id','SalePrice']].to_csv('submission_ridge.csv', index=False)
 sub = simple.Blend(['submission_lgb.csv', 'submission_ridge.csv'], 'Id', 'SalePrice', 'blend' + str(i+2 ).zfill(2)+ '.csv' ).BLEND.reset_index()
<compute_train_metric><EOS> 
testing_linear =