Kaggle competition: Ghouls, Goblins, and Ghosts... Boo!
<SOS> metric: categorizationaccuracy Kaggle data source: ghouls,-goblins,-and-ghosts...-boo!<set_options>
InteractiveShell.ast_node_interactivity='all'
warnings.filterwarnings("ignore")
torch.cuda.empty_cache()<import_modules>
train=pd.read_csv(".. /input/ghouls-goblins-and-ghosts-boo/train.csv")
 test=pd.read_csv(".. /input/ghouls-goblins-and-ghosts-boo/test.csv" )
sys.path.append(".. /input/mynlpscripts/")
 sys.path.extend(['.. /input/bert-utils/'])
<import_modules>
train.drop('id',axis=1,inplace=True)
 test1=test.drop('id',axis=1)
print("TF version:", tf.__version__)
 print("Hub version:", hub.__version__)
<set_options>
train.isnull().sum()
log = logging.getLogger('tensorflow')
 log.handlers = []<feature_engineering>
train['type'].value_counts()
os.environ['TF_FORCE_GPU_ALLOW_GROWTH'] = "true"
StratifiedKFold, GroupShuffleSplit,
GroupKFold, StratifiedShuffleSplit)
 seed(42)
 tf.set_random_seed(42)
 random.seed(42 )<import_modules>
label=LabelEncoder()
train['color']=label.fit_transform(train['color'])
 train['type']=label.fit_transform(train['type'])
 test1['color']=label.fit_transform(test1['color'])
BaseShuffleSplit, _validate_shuffle_split
def IterativeStratification(labels, r, random_state):

 n_samples = labels.shape[0]
test_folds = np.zeros(n_samples, dtype=int)
 c_folds = r * n_samples
c_folds_labels = np.outer(r, labels.sum(axis=0))
labels_not_processed_mask = np.ones(n_samples, dtype=bool)
 while np.any(labels_not_processed_mask):
num_labels = labels[labels_not_processed_mask].sum(axis=0)
 if num_labels.sum() == 0:
sample_idxs = np.where(labels_not_processed_mask)[0]
for sample_idx in sample_idxs:
fold_idx = np.where(c_folds == c_folds.max())[0]
if fold_idx.shape[0] > 1:
fold_idx = fold_idx[random_state.choice(fold_idx.shape[0])]
test_folds[sample_idx] = fold_idx
c_folds[fold_idx] -= 1
break
label_idx = np.where(num_labels == num_labels[np.nonzero(num_labels)].min())[0]
if label_idx.shape[0] > 1:
label_idx = label_idx[random_state.choice(label_idx.shape[0])]
sample_idxs = np.where(np.logical_and(labels[:, label_idx].flatten() , labels_not_processed_mask)) [0]
for sample_idx in sample_idxs:
label_folds = c_folds_labels[:, label_idx]
fold_idx = np.where(label_folds == label_folds.max())[0]
if fold_idx.shape[0] > 1:
temp_fold_idx = np.where(c_folds[fold_idx] ==
c_folds[fold_idx].max())[0]
fold_idx = fold_idx[temp_fold_idx]
if temp_fold_idx.shape[0] > 1:
fold_idx = fold_idx[random_state.choice(temp_fold_idx.shape[0])]
test_folds[sample_idx] = fold_idx
labels_not_processed_mask[sample_idx] = False
c_folds_labels[fold_idx, labels[sample_idx]] -= 1
c_folds[fold_idx] -= 1
return test_folds
class MultilabelStratifiedKFold(_BaseKFold):

 def __init__(self, n_splits=3, shuffle=False, random_state=None):
super(MultilabelStratifiedKFold, self ).__init__(n_splits, shuffle, random_state)
 def _make_test_folds(self, X, y):
y = np.asarray(y, dtype=bool)
 type_of_target_y = type_of_target(y)
 if type_of_target_y != 'multilabel-indicator':
raise ValueError(
 'Supported target type is: multilabel-indicator.Got {!r} instead.'.format(type_of_target_y))
num_samples = y.shape[0]
rng = check_random_state(self.random_state)
 indices = np.arange(num_samples)
 if self.shuffle:
rng.shuffle(indices)
 y = y[indices]
r = np.asarray([1 / self.n_splits] * self.n_splits)
 test_folds = IterativeStratification(labels=y, r=r, random_state=rng)
 return test_folds[np.argsort(indices)]
def _iter_test_masks(self, X=None, y=None, groups=None):
test_folds = self._make_test_folds(X, y)
 for i in range(self.n_splits):
yield test_folds == i
def split(self, X, y, groups=None):

 y = check_array(y, ensure_2d=False, dtype=None)
 return super(MultilabelStratifiedKFold, self ).split(X, y, groups)
 class RepeatedMultilabelStratifiedKFold(_RepeatedSplits):

 def __init__(self, n_splits=5, n_repeats=10, random_state=None):
super(RepeatedMultilabelStratifiedKFold, self ).__init__(
 MultilabelStratifiedKFold, n_repeats, random_state,
n_splits=n_splits)
 class MultilabelStratifiedShuffleSplit(BaseShuffleSplit):

 def __init__(self, n_splits=10, test_size="default", train_size=None,
random_state=None):
super(MultilabelStratifiedShuffleSplit, self ).__init__(
 n_splits, test_size, train_size, random_state)
 def _iter_indices(self, X, y, groups=None):
n_samples = _num_samples(X)
 y = check_array(y, ensure_2d=False, dtype=None)
 y = np.asarray(y, dtype=bool)
 type_of_target_y = type_of_target(y)
 if type_of_target_y != 'multilabel-indicator':
raise ValueError(
 'Supported target type is: multilabel-indicator.Got {!r} instead.'.format(
 type_of_target_y))
n_train, n_test = _validate_shuffle_split(n_samples, self.test_size,
self.train_size)
 n_samples = y.shape[0]
rng = check_random_state(self.random_state)
 y_orig = y.copy()
r = np.array([n_train, n_test])/(n_train + n_test)
 for _ in range(self.n_splits):
indices = np.arange(n_samples)
 rng.shuffle(indices)
 y = y_orig[indices]
test_folds = IterativeStratification(labels=y, r=r, random_state=rng)
 test_idx = test_folds[np.argsort(indices)] == 1
test = np.where(test_idx)[0]
train = np.where(~test_idx)[0]
yield train, test
def split(self, X, y, groups=None):

 y = check_array(y, ensure_2d=False, dtype=None)
 return super(MultilabelStratifiedShuffleSplit, self ).split(X, y, groups )<groupby>
train.drop('color',axis=1,inplace=True)
 test1.drop('color',axis=1,inplace=True )
def stratified_group_k_fold(X, y, groups, k, seed=None):
labels_num = np.max(y)+ 1
y_counts_per_group = defaultdict(lambda: np.zeros(labels_num))
y_distr = Counter()
for label, g in zip(y, groups):
y_counts_per_group[g][label] += 1
y_distr[label] += 1
y_counts_per_fold = defaultdict(lambda: np.zeros(labels_num))
groups_per_fold = defaultdict(set)
 def eval_y_counts_per_fold(y_counts, fold):
y_counts_per_fold[fold] += y_counts
std_per_label = []
for label in range(labels_num):
label_std = np.std([y_counts_per_fold[i][label] / y_distr[label] for i in range(k)])
 std_per_label.append(label_std)
 y_counts_per_fold[fold] -= y_counts
return np.mean(std_per_label)
 groups_and_y_counts = list(y_counts_per_group.items())
 random.Random(seed ).shuffle(groups_and_y_counts)
 for g, y_counts in sorted(groups_and_y_counts, key=lambda x: -np.std(x[1])) :
best_fold = None
min_eval = None
for i in range(k):
fold_eval = eval_y_counts_per_fold(y_counts, i)
 if min_eval is None or fold_eval < min_eval:
min_eval = fold_eval
best_fold = i
y_counts_per_fold[best_fold] += y_counts
groups_per_fold[best_fold].add(g)
 all_groups = set(groups)
 for i in range(k):
train_groups = all_groups - groups_per_fold[i]
test_groups = groups_per_fold[i]
train_indices = [i for i, g in enumerate(groups)if g in train_groups]
test_indices = [i for i, g in enumerate(groups)if g in test_groups]
yield train_indices, test_indices<categorify>
X=train.drop('type',axis=1)
 y=train['type']
def chunks(l, n):

 for i in range(0, len(l), n):
yield l[i:i + n]<categorify>
Classification_models=[('LogisticRegression',LogisticRegression()),('StochasticGDC',SGDClassifier()),('KNC',KNeighborsClassifier()),('SVC',SVC()), 
('LinearSVC',LinearSVC()),('GNaiveBayes',GaussianNB()),('MNaiveBayes',MultinomialNB()),('DTree',DecisionTreeClassifier()), 
('MLPerceptronC',MLPClassifier()),('RF',RandomForestClassifier()),('ET',ExtraTreesClassifier()),('AdaBoostC',AdaBoostClassifier()), 
('GBC',GradientBoostingClassifier()),('XGBC',XGBClassifier())]
def build_module_fn(config_path, vocab_path, do_lower_case=True):
def bert_module_fn(is_training):

 input_ids = tf.placeholder(shape=[None, None], dtype=tf.int32, name="input_ids")
 input_mask = tf.placeholder(shape=[None, None], dtype=tf.int32, name="input_mask")
 token_type = tf.placeholder(shape=[None, None], dtype=tf.int32, name="segment_ids")
 config = BertConfig.from_json_file(config_path)
 model = BertModel(config=config, is_training=is_training,
input_ids=input_ids, input_mask=input_mask, token_type_ids=token_type)
 seq_output = model.all_encoder_layers[-1]
pool_output = model.get_pooled_output()
config_file = tf.constant(value=config_path, dtype=tf.string, name="config_file")
 vocab_file = tf.constant(value=vocab_path, dtype=tf.string, name="vocab_file")
 lower_case = tf.constant(do_lower_case)
 tf.add_to_collection(tf.GraphKeys.ASSET_FILEPATHS, config_file)
 tf.add_to_collection(tf.GraphKeys.ASSET_FILEPATHS, vocab_file)
 input_map = {"input_ids": input_ids,
"input_mask": input_mask,
"segment_ids": token_type}
output_map = {"pooled_output": pool_output,
"sequence_output": seq_output}
output_info_map = {"vocab_file": vocab_file,
"do_lower_case": lower_case}
hub.add_signature(name="tokens", inputs=input_map, outputs=output_map)
 hub.add_signature(name="tokenization_info", inputs={}, outputs=output_info_map)
 return bert_module_fn<find_best_params>
result=[]
names=[]
for name,model in Classification_models:
cvresult=cross_val_score(model,X,y,cv=5,n_jobs=-1,scoring = 'accuracy')
 result.append(cvresult.mean())
 names.append(name)
 print("%s gives %f " %(name, cvresult.mean()))
MODEL_DIR1 = '.. /input/bertbase'
config_path1 = "{}/config.json".format(MODEL_DIR1)
 vocab_path1 = "{}/vocab.txt".format(MODEL_DIR1)
 tags_and_args = []
for is_training in(True, False):
tags = set()
if is_training:
tags.add("train")
 tags_and_args.append(( tags, dict(is_training=is_training)))
 module_fn = build_module_fn(config_path1, vocab_path1)
 spec = hub.create_module_spec(module_fn, tags_and_args=tags_and_args)
 spec.export("bert-module1",
checkpoint_path="{}/bert_model.ckpt".format(MODEL_DIR1))<find_best_params>
params={'C':[0.01,0.1,1],'gamma':[1,0.1,0.01],'kernel':['linear', 'poly', 'rbf'] }
grid=GridSearchCV(SVC() ,param_grid=params,n_jobs=-1,cv=5)
 gridfit=grid.fit(X,y )
MODEL_DIR = '.. /input/bertlarge'
config_path = "{}/bert_config.json".format(MODEL_DIR)
 vocab_path = "{}/vocab.txt".format(MODEL_DIR)
 tags_and_args = []
for is_training in(True, False):
tags = set()
if is_training:
tags.add("train")
 tags_and_args.append(( tags, dict(is_training=is_training)))
 module_fn = build_module_fn(config_path, vocab_path)
 spec = hub.create_module_spec(module_fn, tags_and_args=tags_and_args)
 spec.export("bert-module",
checkpoint_path="{}/bert_model.ckpt".format(MODEL_DIR))<load_from_csv>
gridfit.best_score_
gridfit.best_params_
PATH = '.. /input/google-quest-challenge/'
BERT_PATH = '.. /input/bert-tfhub'
USE_PATH = '.. /input/uselarge4/'
tokenizer = FullTokenizer(vocab_path, True)
 tokenizer1 = FullTokenizer(vocab_path1, True)
 MAX_SEQUENCE_LENGTH = 512
MAX_SEQUENCE_LENGTH_t = 120
MAX_SEQUENCE_LENGTH_q = 416
MAX_SEQUENCE_LENGTH_a = 416
df_train = pd.read_csv(PATH+'train.csv')
 df_test = pd.read_csv(PATH+'test.csv')
 df_sub = pd.read_csv(PATH+'sample_submission.csv')
 print('train shape =', df_train.shape)
 print('test shape =', df_test.shape)
 output_categories = list(df_train.columns[11:])
 input_categories = list(df_train.columns[[1,2,5]])
 print('
output categories:
\t', output_categories)
 print('
input categories:
\t', input_categories )<categorify>
vc=VotingClassifier(estimators=[('Support Vector Classifier',SVC(C=1, gamma=1, kernel='linear')) , 
('Gaussian Naive Bayes',GaussianNB())])
 vc.fit(X,y)
 predictions = vc.predict(test1 )
def read_examples(str_list):

 unique_id = 0
for s in str_list:
line = convert_to_unicode(s)
 if not line:
continue
line = line.strip()
text_a = None
text_b = None
m = re.match(r"^ (.*)\|\|\| (.*)$", line)
 if m is None:
text_a = line
else:
text_a = m.group(1)
 text_b = m.group(2)
 yield InputExample(unique_id=unique_id, text_a=text_a, text_b=text_b)
 unique_id += 1<categorify>
submission = pd.DataFrame({'id':test['id'], 'type':predictions} )
def features_to_arrays(features):
all_input_ids = []
all_input_mask = []
all_segment_ids = []
for feature in features:
all_input_ids.append(feature.input_ids)
 all_input_mask.append(feature.input_mask)
 all_segment_ids.append(feature.input_type_ids)
 return(np.array(all_input_ids, dtype='int32'),
np.array(all_input_mask, dtype='int32'),
np.array(all_segment_ids, dtype='int32'))<categorify>
submission['type']=label.inverse_transform(submission['type'] )
def build_preprocessor(voc_path, seq_len, lower=True):
tokenizer = FullTokenizer(vocab_file=voc_path, do_lower_case=lower)
 def strings_to_arrays(sents):
sents = np.atleast_1d(sents ).reshape(( -1,))
examples = []
for example in read_examples(sents):
examples.append(example)
 features = convert_examples_to_features(examples, seq_len, tokenizer)
 arrays = features_to_arrays(features)
 return arrays
return strings_to_arrays<define_variables>
submission.to_csv('submission.csv', index=False )
targets = [
'question_asker_intent_understanding',
'question_body_critical',
'question_conversational',
'question_expect_short_answer',
'question_fact_seeking',
'question_has_commonly_accepted_answer',
'question_interestingness_others',
'question_interestingness_self',
'question_multi_intent',
'question_not_really_a_question',
'question_opinion_seeking',
'question_type_choice',
'question_type_compare',
'question_type_consequence',
'question_type_definition',
'question_type_entity',
'question_type_instructions',
'question_type_procedure',
'question_type_reason_explanation',
'question_type_spelling',
'question_well_written',
'answer_helpful',
'answer_level_of_information',
'answer_plausible',
'answer_relevance',
'answer_satisfaction',
'answer_type_instructions',
'answer_type_procedure',
'answer_type_reason_explanation',
'answer_well_written'
]
input_columns = ['question_title', 'question_body', 'answer'] 
<categorify>
submission.to_csv('submission.csv', index=False )