Kaggle competition: TMDB Box Office Prediction
nan_pipeline = make_pipeline(ZerofillRezEscOutOfBounds() , ZerofillV18q1ForFalseV18Q() , ZerofillV2a1())
 train_nan = nan_pipeline.fit_transform(train.copy())
 print_nan_counts(train_nan )<categorify>
plt.style.use('ggplot')
 %matplotlib inline
class TransformYesNoToNumeric(BaseEstimator, TransformerMixin):

 def __init__(self):
pass
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
replacements = {'yes': 1, 'no': 0}
columns = [('edjefe', 'uint8'), 
('edjefa', 'uint8'), 
('dependency', 'float16')]
for column, converted_type in columns:
if X[column].dtype == 'object':
X[column] = X[column].replace(replacements ).astype(converted_type)
 return X
yes_no_transformer = TransformYesNoToNumeric()
train_yes_no_transformed = yes_no_transformer.transform(train.copy())
 assert train_yes_no_transformed.select_dtypes(include='object' ).columns.values.tolist() == ['Id', 'idhogar'], 'There are columns aside from Id and idhogar that are type object.'<train_model>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 train.shape, test.shape
class AggregateIndividualFeatures(BaseEstimator, TransformerMixin):

 def __init__(self):
self.excluded_individual_features = ['v18q', 'male', 'female', 'Target']
self.individual_features_to_aggregate = [feature for feature in individual_features if feature not in self.excluded_individual_features]
self.aggregations = ['sum', 'min', 'max', 'std', 'mean', 'median']
self.aggregated_features = []
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
if 'instlevel' in X.columns and 'instlevel' not in self.individual_features_to_aggregate:
self.individual_features_to_aggregate.append('instlevel')
 aggregates = X[id_features + self.individual_features_to_aggregate].groupby("idhogar" ).agg(self.aggregations)
 aggregates.columns = ['-'.join(column ).strip() for column in aggregates.columns.values]
self.aggregated_features = aggregates.columns.values.tolist()
return X.merge(aggregates, on='idhogar', how='left')
 ind_aggregator = AggregateIndividualFeatures()
train_ind_aggregated = ind_aggregator.transform(train.copy())
 household_id = '6893e65ca'
for aggregation in ind_aggregator.aggregations:
manual = train.loc[train.idhogar == household_id].age.apply(aggregation)
 aggregated = train_ind_aggregated.loc[(train_ind_aggregated.idhogar == household_id), [f"age-{aggregation}"]].iloc[0].values[0]
assert manual == aggregated, f"Calculated {aggregation} for age doesn't match aggregation for household {household_id}"
print(f"Calculated {aggregation} for age, {aggregated}, matches manual aggregation, {manual}, for household {household_id}" )<train_model>
def refmt_str2dict(df, cols):
for col in cols:
df[col] = df[col].apply(lambda x: {} if pd.isna(x)else ast.literal_eval(x))
return df
dict_columns = ['belongs_to_collection', 'genres', 'production_companies', 'production_countries',
'spoken_languages', 'Keywords', 'cast', 'crew']
train = refmt_str2dict(train, dict_columns )
class Ordinalizer(BaseEstimator, TransformerMixin):

 def __init__(self):
pass
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
features = ['instlevel', 'epared', 'etecho', 'eviv']
for feature in features:
columns = [column for column in all_features if column.startswith(feature)]
X[feature] = X.loc[:, columns].idxmax(1 ).apply(lambda x: columns.index(x)+ 1)
 if feature != 'instlevel':
X.drop(columns, axis=1, inplace=True)
 return X
ordinalizer = Ordinalizer()
train_ordinalized = ordinalizer.transform(train.copy())
 for index, individual in train_ordinalized.sample(10 ).iterrows() :
for feature in ['instlevel', 'epared', 'etecho', 'eviv']:
assert train.loc[index, f"{feature}{individual[feature]}"] == 1, f"Ordinalized {feature} doesn't match original boolean instlevel"
for feature in ['epared', 'etecho', 'eviv']:
feature_columns = [column for column in train_ordinalized.columns if column.startswith(feature)]
assert len(feature_columns)== 1, f"{feature} column is still present."
household_id = "6893e65ca"
instlevel_ind_aggregator = AggregateIndividualFeatures()
train_instlevel_ind_aggregated = instlevel_ind_aggregator.transform(train_ordinalized)
 assert 'instlevel-sum' in instlevel_ind_aggregator.aggregated_features, "instlevel is not in aggregated features."
for aggregation in instlevel_ind_aggregator.aggregations:
manual = train_ordinalized.loc[train_ordinalized.idhogar == household_id].instlevel.apply(aggregation)
 aggregated = train_instlevel_ind_aggregated.loc[(train_instlevel_ind_aggregated.idhogar == household_id), [f"instlevel-{aggregation}"]].iloc[0].values[0]
assert manual == aggregated, f"Calculated {aggregation} for instlevel doesn't match aggregation for household {household_id}"
print(f"Calculated {aggregation} for instlevel, {aggregated}, matches manual aggregation, {manual}, for household {household_id}" )<prepare_x_and_y>
TMDB_path = 'https://image.tmdb.org/t/p/w600_and_h900_bestv2/'
class FeatureCreator(BaseEstimator, TransformerMixin):

 def __init__(self):
self.created_features = []
def fit(self, X, y=None):
self.created_features = []
self._add_feature('rent_per_room', lambda X: X.v2a1 / X.rooms)
 self._add_feature('rent_per_hhsize', lambda X: X.v2a1 / X.hhsize)
 self._add_feature('tablets_per_hhsize', lambda X: X.v18q1 / X.hhsize)
 self._add_feature('tablets_per_adult', lambda X: X.v18q1 / X.hogar_adul)
 self._add_feature('escolari-mean_to_age_mean', lambda X: X['escolari-mean'] / X['age-mean'])
 self._add_feature('rez_esc-mean_to_age_mean', lambda X: X['rez_esc-mean'] / X['age-mean'])
 self._add_feature('males_to_females', lambda X: X.r4h3 / X.r4m3)
 self._add_feature('under12_to_over12', lambda X: X.r4t1 / X.r4t2)
 return self
def transform(self, X, y=None):
for label, calculation in self.created_features:
X[label] = calculation(X)
 X.replace(np.inf, np.nan, inplace=True)
 return X
def _add_feature(self, label, calculation):
self.created_features.append(( label, calculation))
ind_aggregator = AggregateIndividualFeatures()
train_ind_aggregated = ind_aggregator.transform(train.copy())
 feature_creator = FeatureCreator()
features_created = feature_creator.fit_transform(train_ind_aggregated.copy())
 assert features_created.rent_per_room.equals(features_created.v2a1 / features_created.rooms), "rent_per_room wasn't created correctly."
assert len(features_created.columns)- len(train_ind_aggregated.columns)== 8, "Didn't create the expected number of new features"
features_created[['rent_per_room', 'v2a1', 'rooms']].head()
assert len(features_created.columns[features_created[features_created == np.inf].any() ])== 0, "inifinity values present"<groupby>
def fix_date(x):
yr = x.split('/')[2]
if int(yr)<= 19:
return x[:-2] + '20' + yr
else:
return x[:-2] + '19' + yr
train['release_date'] = train['release_date'].apply(lambda x: fix_date(x))
train['release_date'] = pd.to_datetime(train['release_date'] )
household_groups = train.groupby('idhogar')
 assert len(household_groups)== len(train.idhogar.unique()), "Length of household groups is not the same as the number of unique household ids."
assert not household_groups[["idhogar"] + household_features].var().any().any() , "Not all rows for a group have the same values for a household feature"<categorify>
def process_date(df):
date_attrs = ['year', 'month', 'day', 'weekday', 'weekofyear', 'quarter']
for attr in date_attrs:
new_col = 'release_date_' + attr
df[new_col] = getattr(df['release_date'].dt, attr ).astype(int)
 return df
train = process_date(train )
class ExtractHouseholds(BaseEstimator, TransformerMixin):

 def __init__(self, individual_aggregator=None, feature_creator=None):
self.individual_aggregator = individual_aggregator
self.feature_creator = feature_creator
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
self.included_features = household_features + squared_household_features
if self.individual_aggregator:
self.included_features = self.included_features + self.individual_aggregator.aggregated_features
if self.feature_creator:
self.included_features = self.included_features + [feature for feature, _ in self.feature_creator.created_features]
self.included_features = [feature for feature in self.included_features if feature in X]
return X.groupby('idhogar')[self.included_features].first()
household_extractor = ExtractHouseholds()
train_households = household_extractor.transform(train.copy())
 assert len(train_households)== len(train.idhogar.unique()), f"Extracted households length({len(train_households)})doesn't match unique household indentifiers in train set({len(train.idhogar.unique())} )."
household_extractor = ExtractHouseholds(individual_aggregator=instlevel_ind_aggregator)
 train_households = household_extractor.transform(train_instlevel_ind_aggregated.copy())
 assert len(train_households)== len(train_instlevel_ind_aggregated.idhogar.unique()), f"Extracted households length({len(train_households)})doesn't match unique household indentifiers in train set({len(train_instlevel_ind_aggregated.idhogar.unique())} )."
train_households.head()<import_modules>
def WordCloud_fromDict(dict_name, key='name'):
list_dict = list(train[dict_name].apply(lambda x: [i[key] for i in x] if x != {} else [] ).values)
 list2txt = ' '.join(['_'.join(i.split(' ')) for j in list_dict for i in j])
 wordcloud = WordCloud(max_font_size = None, background_color = 'black', collocations = False,
width = 1200, height = 1000 ).generate(list2txt)
 return wordcloud
import scipy
from sklearn.impute import SimpleImputer
from sklearn.preprocessing import MinMaxScaler, StandardScaler
from sklearn.model_selection import cross_validate, RandomizedSearchCV
from sklearn.metrics.scorer import make_scorer
from sklearn.metrics import f1_score
from sklearn.tree import DecisionTreeClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier, AdaBoostClassifier
from sklearn.linear_model import SGDClassifier
from sklearn.neural_network import MLPClassifier<categorify>
list_genres = list(train['genres'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 list_keywords = list(train['Keywords'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 genres = set([y for x in list_genres for y in x])
 keywords = set([y for x in list_keywords for y in x])
 len(genres), len(keywords )
class FeatureSelector(BaseEstimator, TransformerMixin):

 def __init__(self, selected_features=[], excluded_features=[]):
self.selected_features = selected_features
self.excluded_features = excluded_features
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
if not self.selected_features:
self.selected_features = X.columns.tolist()
if self.excluded_features:
self.selected_features = [feature for feature in self.selected_features if feature not in self.excluded_features]
return X[self.selected_features]<choose_model_class>
list_crew = list(train['crew'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 cast = set([y for x in list_cast_wURL for y in x])
 crew = set([y for x in list_crew for y in x])
 len(cast), len(crew )
class ModelEvaluator() :
def __init__(self, pipeline, estimators, features=[], excluded_features=[]):
self.pipeline = pipeline
self.estimators = [self._init_estimator(estimator)for estimator in estimators]
self._preprocessed = False
self.features = features
self.excluded_features = excluded_features
self.train, self.test = import_train_test()
self.results = []
def _init_estimator(self, estimator):
return {'estimator': estimator,
'fit': False,
'cv_results': {},
'feature_importances': None,
'tuned': False,
'tuned_best_estimator': None,
'tuned_best_score': None,
'tuned_best_params': None,
'tuned_cv_results': {}}
def _preprocess_data(self):
try:
self.pipeline.named_steps['featureselector'].selected_features = self.features
self.pipeline.named_steps['featureselector'].excluded_features = self.excluded_features
except KeyError:
pass
self._trainX_preprocessed = pipeline.fit_transform(self.train.copy())
 self._trainY_preprocessed = self.train.copy().groupby('idhogar')['Target'].first()
self._testX_preprocessed = self.pipeline.transform(self.test.copy())
 self._preprocessed = True
def evaluate(self):
if not self._preprocessed:
self._preprocess_data()
print(f"Evaluating on {self._trainX_preprocessed.shape[1]} features and {self._trainX_preprocessed.shape[0]} samples.")
 print("")
 for estimator in self.estimators:
estimator['cv_results'] = self._cross_validate(estimator)
 estimator['feature_importances'] = self._calculate_feature_importances(estimator)
 self._print_cv_results()
def _cross_validate(self, estimator):
return cross_validate(estimator['estimator'],
self._trainX_preprocessed,
self._trainY_preprocessed,
cv=5,
scoring='f1_macro',
n_jobs=-1,
return_train_score=False)
 def _fit_estimator(self, estimator):
estimator['estimator'].fit(self._trainX_preprocessed, self._trainY_preprocessed)
 estimator['fit'] = True
def _calculate_feature_importances(self, estimator):
if not estimator['fit']:
self._fit_estimator(estimator)
 try:
return self._feature_importances_dataframe(estimator['estimator'].feature_importances_)
 except AttributeError:
return None
def _feature_importances_dataframe(self, feature_importances):
trained_features = self.pipeline.named_steps['featureselector'].selected_features
return pd.DataFrame(data={'importance': feature_importances},
index=trained_features ).sort_values(by='importance', ascending=False)
 def _print_cv_results(self):
for estimator in self.estimators:
print(f"Scores for {estimator['estimator']}")
 print(f"Mean Macro F1 Score: {estimator['cv_results']['test_score'].mean() }, SD={estimator['cv_results']['test_score'].std() }")
 print("")
 def tune_hyperparameters(self, estimator, param_dist, n_iter=20, cv=5):
random_search = RandomizedSearchCV(estimator['estimator'],
param_distributions=param_dist,
n_iter=n_iter,
cv=cv,
scoring='f1_macro',
n_jobs=-1,
verbose=1)
 random_search.fit(self._trainX_preprocessed, self._trainY_preprocessed)
 estimator['tuned'] = True
estimator['tuned_best_estimator'] = random_search.best_estimator_
estimator['tuned_best_score'] = random_search.best_score_
estimator['tuned_cv_results'] = random_search.cv_results_
estimator['tuned_best_params'] = random_search.best_params_
estimator['tuned_best_index'] = random_search.best_index_
return random_search
def prepare_submissions(self):

 for estimator in self.estimators:
if not estimator['tuned']:
continue
self._save_submission(estimator)
 def _save_submission(self, estimator):
if not estimator['tuned_best_estimator']:
pass
predictions = self.test.copy().groupby('idhogar' ).first().reset_index()
predictions['Target'] = estimator['tuned_best_estimator'].predict(self._testX_preprocessed)
 test_results = test[['Id', 'idhogar']].copy()
test_results = test_results.merge(predictions[['idhogar', 'Target']].copy() , on="idhogar", how="left" ).drop("idhogar", axis=1)
 assert test_results.shape[0] == self.test.shape[0], "Number of results don't match number of test samples."
filename = f"{estimator['tuned_best_estimator'].__class__.__name__}-tuned-{estimator['tuned_best_score']:.3f}.csv"
self._save_predictions(test_results, filename)
 def _save_predictions(self, predictions, filename):

 predictions.to_csv(filename, index=False)
 print(f"Wrote results to {filename}" )<define_search_model>
list_prodCompanies = list(train['production_companies'].apply(
 lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 list_prodCountries = list(train['production_countries'].apply(
 lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 prodCompanies = set([y for x in list_prodCompanies for y in x])
 prodCountries = set([y for x in list_prodCountries for y in x])
 len(prodCompanies), len(prodCountries )
individual_aggregator = AggregateIndividualFeatures()
feature_creator = FeatureCreator()
pipeline = make_pipeline(ZerofillRezEscOutOfBounds() ,
ZeroMaxRezEsc() ,
ZerofillV18q1ForFalseV18Q() ,
ZerofillV2a1() ,
TransformYesNoToNumeric() ,
Ordinalizer() ,
individual_aggregator,
feature_creator,
ExtractHouseholds(individual_aggregator=individual_aggregator,
feature_creator=feature_creator),
FeatureSelector() ,
SimpleImputer() ,
MinMaxScaler() 
)
 estimators = [DecisionTreeClassifier(random_state=42),
KNeighborsClassifier() ,
LogisticRegression(random_state=42, solver='lbfgs', multi_class='multinomial', max_iter=1000),
RandomForestClassifier(random_state=42, n_estimators=100),
AdaBoostClassifier(random_state=42),
GradientBoostingClassifier(random_state=42),
SGDClassifier(random_state=42, max_iter=1000, tol=1e-3),
MLPClassifier(random_state=42, solver='lbfgs')]
evaluator = ModelEvaluator(pipeline, estimators)
 evaluator.evaluate()<choose_model_class>
OneMovie = train[train['title'] == 'The Terminator']
keywords = list(OneMovie['Keywords'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)[0]
keywords_str = ' '.join(keywords)
 text_merged = list(OneMovie['overview'])[0] + ' | ' + list(OneMovie['tagline'])[0] + ' | ' + keywords_str
text_merged
pipeline = make_pipeline(ZerofillRezEscOutOfBounds() ,
ZeroMaxRezEsc() ,
ZerofillV18q1ForFalseV18Q() ,
ZerofillV2a1() ,
TransformYesNoToNumeric() ,
Ordinalizer() ,
individual_aggregator,
feature_creator,
ExtractHouseholds(individual_aggregator=individual_aggregator,
feature_creator=feature_creator)
)
 evaluator_corr = ModelEvaluator(pipeline, estimators)
 evaluator_corr._preprocess_data()
corr = evaluator_corr._trainX_preprocessed.corr()<train_model>
text_merged2train = []
for id in train['id']:
mov = train[train['id'] == id]
keywords = list(mov['Keywords'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)[0]
keywords_str = ' '.join(keywords)
 if pd.isnull(list(mov['overview'])[0]):
overview = ''
else:
overview = list(mov['overview'])[0]
if pd.isnull(list(mov['tagline'])[0]):
tagline = ''
else:
tagline = list(mov['tagline'])[0]
text_merged = keywords_str
text_merged2train.append(text_merged)
 train['text_merged'] = text_merged2train
print(evaluator_corr.train.columns)
 print(evaluator_corr._trainX_preprocessed.columns )<filter>
train['belongs_to_collection'].apply(lambda x: 1 if x != {} else 0 ).value_counts()
for attribute in corr.loc[correlated_features, correlated_features]:
print(corr.loc[attribute, corr.loc[attribute].abs().gt(0.95)])
 print()<filter>
train['belongs2coll_yn'] = train['belongs_to_collection'].apply(lambda x: len(x)if x != {} else 0)
 train['homepage'].head(5 )
variance = len(train.loc[train[['r4t3', 'tamhog', 'hhsize', 'hogar_total']].var(axis=1)!= 0, ['r4t3', 'tamhog', 'hhsize', 'hogar_total']])
 print(f"Variance in train between 'r4t3', 'tamhog', 'hhsize', 'hogar_total': {variance}")
 variance = len(test.loc[test[['r4t3', 'tamhog', 'hhsize', 'hogar_total']].var(axis=1)!= 0, ['r4t3', 'tamhog', 'hhsize', 'hogar_total']])
 print(f"Variance in test between 'r4t3', 'tamhog', 'hhsize', 'hogar_total': {variance}")
 variance = len(train.loc[train[['tamhog', 'hhsize', 'hogar_total']].var(axis=1)!= 0, ['tamhog', 'hhsize', 'hogar_total']])
 print(f"Variance in train between 'tamhog', 'hhsize', 'hogar_total': {variance}")
 variance = len(test.loc[test[['tamhog', 'hhsize', 'hogar_total']].var(axis=1)!= 0, ['tamhog', 'hhsize', 'hogar_total']])
 print(f"Variance in test between 'tamhog', 'hhsize', 'hogar_total': {variance}")
 train.loc[train[['r4t3', 'tamhog', 'hhsize', 'hogar_total']].var(axis=1)!= 0, ['r4t3', 'tamhog', 'hhsize', 'hogar_total','Target']].head(10 )<filter>
train['homepage'].apply(lambda x: 1 if pd.isnull(x)== False else 0 ).value_counts()
print(train.loc[(train.tamviv != train.r4t3), ['tamviv', 'r4t3', 'tamhog', 'hhsize', 'hogar_total', 'Target']].head())
 print(train.loc[(train.tamviv > train.r4t3), ['tamviv', 'r4t3', 'tamhog', 'hhsize', 'hogar_total', 'Target']].head())
 equality = train.loc[(train.tamviv != train.r4t3)].equals(train.loc[(train.tamviv > train.r4t3)])
 print(f"When tamviv isn't equal to r4t3, tamviv is greater than tamviv, train: {equality}")
 equality = test.loc[(test.tamviv != test.r4t3)].equals(test.loc[(test.tamviv > test.r4t3)])
 print(f"When tamviv isn't equal to r4t3, tamviv is greater than tamviv, test: {equality}")
 print(len(train.loc[(train.r4t3 != train.hhsize)&(train.tamviv != train.r4t3), ['tamviv', 'r4t3', 'tamhog', 'hhsize', 'hogar_total', 'Target']]))<categorify>
train['homepage_yn'] = train['homepage'].apply(lambda x: 1 if pd.isnull(x)== False else 0 )
class TamvivR4t3Combined(BaseEstimator, TransformerMixin):

 def __init__(self):
self.created_features = []
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
X['tamviv_r4t3_combined'] = X[['r4t3', 'tamviv']].max(axis=1)
 self.created_features = ['tamviv_r4t3_combined']
return X
tamviv_r4t3_combiner = TamvivR4t3Combined()
tamviv_r4t3_preprocessed = tamviv_r4t3_combiner.transform(train.copy())
 max_check = tamviv_r4t3_preprocessed.tamviv_r4t3_combined.equals(tamviv_r4t3_preprocessed[['r4t3', 'tamviv']].max(axis=1))
assert max_check, "TamvivR4t3Combiner didn't create the correct max column"<define_search_model>
' '.join(set(train['original_language'])) , len(set(train['original_language']))
individual_aggregator = AggregateIndividualFeatures()
feature_creator = FeatureCreator()
pipeline = make_pipeline(ZerofillRezEscOutOfBounds() ,
ZeroMaxRezEsc() ,
ZerofillV18q1ForFalseV18Q() ,
ZerofillV2a1() ,
TransformYesNoToNumeric() ,
Ordinalizer() ,
individual_aggregator,
feature_creator,
ExtractHouseholds(individual_aggregator=individual_aggregator,
feature_creator=feature_creator),
TamvivR4t3Combined() ,
FeatureSelector() ,
SimpleImputer() ,
MinMaxScaler() 
)
 correlated_features_list = correlated_features[correlated_features == True].index
excluded_features = ['elimbasu5', 'area2', 'r4t3', 'tamviv', 'coopele', 'SQBdependency', 'hogar_total', 'tamhog']
kept_features = ['area1', 'public', 'hhsize', 'dependency']
for feature in correlated_features_list:
if feature not in excluded_features + kept_features:
for excluded_feature in corr.loc[feature, corr.loc[feature].abs().between(0.95, 0.9999999)].index:
excluded_features.append(excluded_feature)
 kept_features.append(feature)
 evaluator_excluded = ModelEvaluator(pipeline, estimators, excluded_features=excluded_features)
 evaluator_excluded.evaluate()<features_selection>
train['status'].apply(lambda x: 1 if x == 'Released' else 0 ).value_counts() , set(train['status'] )
def select_important_features(feature_importances, threshold=0.9):
total_importance = 0
important_features = []
for feature, importance in feature_importances.iterrows() :
total_importance += importance[0]
important_features.append(feature)
 if total_importance >= threshold:
break
return important_features<find_best_params>
test = refmt_str2dict(test, dict_columns )
feature_importance_estimator = [estimator for estimator in evaluator_excluded.estimators if estimator['estimator'].__class__.__name__ == 'GradientBoostingClassifier'][0]
important_features = select_important_features(feature_importance_estimator['feature_importances'], threshold=0.95)
 evaluator_important_features = ModelEvaluator(pipeline, estimators, important_features)
 evaluator_important_features.evaluate()<choose_model_class>
test.at[pd.isnull(test['release_date']), 'release_date'] = '1/1/11'
test['release_date'] = test['release_date'].apply(lambda x: fix_date(x))
test['release_date'] = pd.to_datetime(test['release_date'])
 test = process_date(test )
estimators = [GradientBoostingClassifier(random_state=42)]
tuning_evaluator = ModelEvaluator(pipeline, estimators, important_features)
 tuning_evaluator.evaluate()<init_hyperparams>
text_merged2test = []
for id in test['id']:
mov = test[test['id'] == id]
keywords = list(mov['Keywords'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)[0]
keywords_str = ' '.join(keywords)
 if pd.isnull(list(mov['overview'])[0]):
overview = ''
else:
overview = list(mov['overview'])[0]
if pd.isnull(list(mov['tagline'])[0]):
tagline = ''
else:
tagline = list(mov['tagline'])[0]
text_merged = keywords_str
text_merged2test.append(text_merged)
 test['text_merged'] = text_merged2test
param_dist = {'learning_rate': scipy.stats.truncnorm(-0.995, 2, loc=0.1, scale=0.05),
'n_estimators': scipy.stats.randint(80, 1000),
'max_depth': scipy.stats.randint(3, 8),
'min_samples_split': scipy.stats.randint(2, 100),
'min_samples_leaf': scipy.stats.randint(20, 60),
'max_features': ['sqrt', None, 'log2'],
'subsample': scipy.stats.uniform(loc=0.6, scale=0.4)}
results = tuning_evaluator.tune_hyperparameters(tuning_evaluator.estimators[0], param_dist, n_iter=20, cv=5 )<find_best_score>
test['belongs2coll_yn'] = test['belongs_to_collection'].apply(lambda x: len(x)if x != {} else 0)
 test['homepage_yn'] = test['homepage'].apply(lambda x: 1 if pd.isnull(x)== False else 0 )
print(results.best_params_)
 print(results.best_score_ )<find_best_params>
test[test['release_date_year'] == 1927]
tuning_evaluator.prepare_submissions()<set_options>
cols2drop = ['id','belongs_to_collection', 'homepage', 'imdb_id', 'original_title', 'overview',
'poster_path', 'release_date', 'spoken_languages', 'status','tagline', 'Keywords']
train = train.drop(cols2drop, axis=1)
 test = test.drop(cols2drop, axis=1)
 train.columns, test.columns
%matplotlib inline 
<load_from_csv>
df4corr = train[['budget', 'popularity', 'runtime', 'release_date_year', 'release_date_month', 'release_date_day', \
'release_date_weekday', 'release_date_weekofyear', 'release_date_quarter', 'revenue']]
pd.set_option("display.max_rows", None)
 testDf = pd.read_csv('.. /input/test.csv')
 trainDf = pd.read_csv('.. /input/train.csv')
 print("Training dataset basic information")
 print("Rows: {}".format(len(trainDf)))
 print("Columns: {}".format(len(trainDf.columns)))
 trainDf.head()<concatenate>
train['budget_yn'] = train['budget'].apply(lambda x: 0 if x == 0 else 1)
 train['budget_per_year'] = train['budget']/train['release_date_year']
testDf['Target'] = np.nan
data = trainDf.append(testDf, ignore_index = True )<features_selection>
train['n_genres'] = train['genres'].apply(lambda x: len(x)if x != {} else 0)
 train['n_production_companies'] = train['production_companies'].apply(lambda x: len(x)if x != {} else 0)
 train['n_production_countries'] = train['production_countries'].apply(lambda x: len(x)if x != {} else 0)
 train['n_cast'] = train['cast'].apply(lambda x: len(x)if x != {} else 0)
 train['n_crew'] = train['crew'].apply(lambda x: len(x)if x != {} else 0 )
def findColumnsWithNan(df):
cols = df.columns[df.isna().any() ]
print("Number of columns with Nan: {}".format(len(cols)))
 print("Column names: {}".format(cols))
print("-" * 80)
 for col in cols:
print("Column: [{}] missing {} values.".format(col, len(df[df[col].isna() == True])))
 print("Analysis of training dataset...")
 findColumnsWithNan(trainDf )<feature_engineering>
train['genres_collapsed'] = train['genres'].apply(lambda x: ' '.\
join(sorted([i['name'] for i in x])) if x != {} else '')
 for g in genres:
train['genre_' + g] = train['genres_collapsed'].apply(lambda x: 1 if g in x else 0)
 keys_genre = ['genre_' + g for g in genres]
data.loc[(data['tipovivi1'] == 1)&(data['v2a1'].isna()), 'v2a1'] = 0
print("Missing values after replacing: {}".format(len(data.loc[data['v2a1'].isna() ])) )<feature_engineering>
n_prodCompanies = 30
Counter([i for j in list_prodCompanies for i in j] ).most_common(n_prodCompanies )
data.loc[data['v18q1'].isna() , 'v18q1'] = 0
print("Missing values after replacing: {}".format(len(data.loc[data['v18q1'].isna() ])) )<feature_engineering>
train['production_companies_collapsed'] = train['production_companies'].apply(lambda x: ' '.\
join(sorted([i['name'] for i in x])) if x != {} else '')
 top_prodCompanies = [m[0] for m in Counter([i for j in list_prodCompanies for i in j] ).most_common(n_prodCompanies)]
for comp in top_prodCompanies:
train['production_company_' + comp] = train['production_companies_collapsed'].\
apply(lambda x: 1 if comp in x else 0)
 keys_production_company = ['production_company_' + comp for comp in top_prodCompanies]
data.loc[(data['age'] < 7)&(data['rez_esc'].isna()), 'rez_esc'] = 0
data.loc[(data['age'] > 19)&(data['rez_esc'].isna()), 'rez_esc'] = 0
print("Missing values after replacing: {}".format(len(data.loc[data['rez_esc'].isna() ])) )<feature_engineering>
n_prodCountries = 30
Counter([i for j in list_prodCountries for i in j] ).most_common(n_prodCountries )
data.loc[data['age'] < 19 & data['meaneduc'].isna() , 'meaneduc'] = 0
print("Missing values after replacing: {}".format(len(data.loc[data['meaneduc'].isna() ])) )<drop_column>
train['production_countries_collapsed'] = train['production_countries'].apply(lambda x: ' '.\
join(sorted([i['name'] for i in x])) if x != {} else '')
 top_prodCountries = [m[0] for m in Counter([i for j in list_prodCountries for i in j] ).most_common(n_prodCountries)]
for comp in top_prodCountries:
train['production_country_' + comp] = train['production_countries_collapsed'].\
apply(lambda x: 1 if comp in x else 0)
 keys_production_country = ['production_country_' + comp for comp in top_prodCountries]
data.drop('SQBmeaned', inplace=True, axis=1)
 print("Total number of columns left: {}".format(len(data.columns)) )<define_variables>
top_languages = [m[0] for m in Counter(train['original_language'] ).most_common(5)]
for l in top_languages:
train['language_' + l] = train['original_language'].\
apply(lambda x: 1 if l in x else 0)
 keys_language = ['language_' + l for l in top_languages]
for cols in data.columns[1:]:
if cols in ['idhogar', 'dependency', 'edjefe', 'edjefa']:
continue
percentile75 = np.percentile(data[cols].fillna(0), 75)
 percentile25 = np.percentile(data[cols].fillna(0), 25)
 threshold =(percentile75 - percentile25)* 1.5
lower, upper =(percentile25 - threshold),(percentile75 + threshold)
 outliers = data.loc[(data[cols] < lower)&(data[cols] > upper)]
if len(outliers)> 0:
print('Feature: {}.Identified outliers: {}'.format(cols, len(outliers)) )<data_type_conversions>
n_lead = 50
list_lead = list(train['cast'].apply(lambda x: [i['name'] for i in x if i['cast_id'] == 1] if x != {} else [] ).values)
 Counter([i for j in list_lead for i in j] ).most_common(n_lead )
for col in ['dependency', 'edjefe', 'edjefa']:
data.loc[data[col] == 'yes', col] = 1.0
data.loc[data[col] == 'no', col] = 0.0
data[col] = pd.to_numeric(data[col] )<prepare_x_and_y>
fullset = pd.concat([train, test])
 list_lead = list(fullset['cast'].apply(lambda x: [i['name'] for i in x if i['cast_id'] == 1] if x != {} else [] ).values)
 Counter([i for j in list_lead for i in j] ).most_common(n_lead )
def featuresToDrop(corrMatrix):

 upper = corrMatrix.where(np.triu(np.ones(corrMatrix.shape), k=1 ).astype(np.bool))
return [column for column in upper.columns if any(upper[column] > 0.95)]
toDrop = featuresToDrop(corrMat)
 data.drop(toDrop, inplace=True, axis=1)
 print("Correlated features which are dropped: {}".format(toDrop))<merge>
train['cast_collapsed'] = train['cast'].apply(lambda x: ' '.\
join(sorted([i['name'] for i in x])) if x != {} else '')
 top_lead = [m[0] for m in Counter([i for j in list_lead for i in j] ).most_common(n_lead)]
for lead in top_lead:
train['cast_' + lead] = train['cast_collapsed'].apply(lambda x: 1 if lead in x else 0)
 keys_cast = ['cast_' + lead for lead in top_lead]
features = list(data.drop(columns = ['Id', 'idhogar', 'Target'] ).columns)
 aggDf = data.drop(columns='Target' ).groupby('idhogar' ).agg(['min', 'max', 'sum', 'count', 'std'])
 new_col = []
for c in aggDf.columns.levels[0]:
for stat in aggDf.columns.levels[1]:
new_col.append('{}-{}'.format(c, stat))
aggDf.columns = new_col
toDrop = featuresToDrop(aggDf.corr())
 aggDf.drop(toDrop, inplace=True, axis=1)
 data = data.merge(aggDf, on='idhogar', how ='left')
 print('Training feature shape: ', data.shape )<feature_engineering>
train['directors_collapsed'] = train['crew'].apply(lambda x: ' '.\
join(sorted([i['name'] for i in x if i['job'] == 'Director'])) if x != {} else '')
 top_directors = [m[0] for m in Counter([i for j in list_directors for i in j] ).most_common(n_directors)]
for d in top_directors:
train['director_' + d] = train['directors_collapsed'].apply(lambda x: 1 if d in x else 0)
 keys_director = ['director_' + d for d in top_directors]
data['phones-per-capita'] = data['qmobilephone'] / data['tamviv']
data['tablets-per-capita'] = data['v18q1'] / data['tamviv']
data['rooms-per-capita'] = data['rooms'] / data['tamviv']
data['rent-per-capita'] = data['v2a1'] / data['tamviv']<normalization>
trainTarget = np.array(list(data[data['Target'].notnull() ]['Target'].astype(np.uint8)))
 submission = data.loc[data['Target'].isnull() , 'Id'].to_frame()
trainData = data[data['Target'].notnull() ].drop(columns = ['Id', 'idhogar', 'Target'])
 testData = data[data['Target'].isnull() ].drop(columns = ['Id', 'idhogar', 'Target'])
 imputer = SimpleImputer(missing_values=np.nan, strategy='median')
 trainData = imputer.fit_transform(trainData)
 testData = imputer.transform(testData)
 scaler = MinMaxScaler()
trainData = scaler.fit_transform(trainData)
 testData = scaler.transform(testData )<split>
vectorizer = TfidfVectorizer(
 analyzer = 'word',
stop_words = 'english',
ngram_range =(1, 2),
min_df = 10,
sublinear_tf = True)
 overview_transf = vectorizer.fit_transform(train['text_merged'])
 overview_transf
model = lightgbm.LGBMClassifier(max_depth=-1, learning_rate=0.1, objective='multiclass',
random_state=None, silent=True, metric='None',
n_jobs=4, n_estimators=5000, class_weight='balanced',
colsample_bytree = 0.93, min_child_samples = 95, num_leaves = 14, subsample = 0.96)
 kfold = 5
kf = StratifiedKFold(n_splits=kfold, shuffle=True)
 predicts_result = []
for idx,(train_index, test_index)in enumerate(kf.split(trainData, trainTarget)) :
print("Fold: {}".format(idx))
X_train, X_val = trainData[train_index], trainData[test_index]
y_train, y_val = trainTarget[train_index], trainTarget[test_index]
model.fit(X_train, y_train, verbose=100)
 predicts_result.append(model.predict(testData))
submission['Target'] = np.array(predicts_result ).mean(axis=0 ).round().astype(int)
 submission.to_csv('submission.csv', index=False)
 print("Completed!" )<install_modules>
linreg = LinearRegression()
linreg.fit(overview_transf, train['revenue'] )
!pip install torch > /dev/null
!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py > /dev/null
!python pytorch-xla-env-setup.py --version 20200515 --apt-packages libomp5 libopenblas-dev > /dev/null
!pip install transformers > /dev/null
!pip install pandarallel > /dev/null
!pip install emoji
!pip install num2words
!pip install --upgrade language_tool_python
!pip install --upgrade turkishnlp
!pip install --upgrade git+https://github.com/pytorch/contrib.git > /dev/null<set_options>
keywords_newPos = [lexicon[i] for i in poscorrTop]
keywords_newNeg = [lexicon[i] for i in negcorrTop]
keywords_new = keywords_newPos + keywords_newNeg
keywords_new[0:10]
os.environ['XLA_USE_BF16'] = "1"
pandarallel.initialize(nb_workers=8, progress_bar=False)
<define_variables>
for k in keywords_new:
train['txt_' + k] = train['text_merged'].apply(lambda x: 1 if k in x else 0)
 keys_txt = ['txt_' + s for s in keywords_new]
MODEL = 'xlm-roberta-large'
root_dir = Path(".. /input")
 test_file_dir = Path(root_dir, "jigsaw-multilingual-toxic-comment-classification")
 model_dir = Path(root_dir, MODEL)
 checkpoint_dir =(model_dir/'best_model.bin')
 config_path =(model_dir/'config.json')
 out_dir = Path('.. /output')
 test_file = "test.csv"
test_eng_file1 = "jigsaw-multilingual-toxic-test-translated/jigsaw_miltilingual_test_translated.csv"
test_eng_file2 = "test-en-df/test_en.csv"
sub_file = "jigsaw-toxic/sample_submission.csv"
open_subtitles_file = 'open-subtitles-toxic/open-subtitles-synthesic.csv'
LANGS = {
'en': 'english',
'it': 'italian',
'fr': 'french',
'es': 'spanish',
'tr': 'turkish',
'ru': 'russian',
'pt': 'portuguese'
}
with open(config_path)as f:
model_config = json.load(f)
 MODEL_VERSION = model_config['model_version']
MODEL_PREFIX = model_config['model_prefix']
MAX_LENGTH = model_config['max_len']
english_pipeline = model_config['english pipeline']
output_hidden_states = model_config['output_hidden_states']<categorify>
test['budget_yn'] = test['budget'].apply(lambda x: 0 if x == 0 else 1)
 test['budget_per_year'] = test['budget']/test['release_date_year']
test['n_genres'] = test['genres'].apply(lambda x: len(x)if x != {} else 0)
 test['n_production_companies'] = test['production_companies'].apply(lambda x: len(x)if x != {} else 0)
 test['n_production_countries'] = test['production_countries'].apply(lambda x: len(x)if x != {} else 0)
 test['n_cast'] = test['cast'].apply(lambda x: len(x)if x != {} else 0)
 test['n_crew'] = test['crew'].apply(lambda x: len(x)if x != {} else 0)
 for l in top_languages:
test['language_' + l] = test['original_language'].\
apply(lambda x: 1 if l in x else 0)
 test['genres_collapsed'] = test['genres'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 for g in genres:
test['genre_' + g] = test['genres_collapsed'].apply(lambda x: 1 if g in x else 0)
 test['production_companies_collapsed'] = test['production_companies'].apply(lambda x: ' '.\
join(sorted([i['name'] for i in x])) if x != {} else '')
 for comp in top_prodCompanies:
test['production_company_' + comp] = test['production_companies_collapsed'].\
apply(lambda x: 1 if comp in x else 0)
 test['production_countries_collapsed'] = test['production_countries'].apply(lambda x: ' '.\
join(sorted([i['name'] for i in x])) if x != {} else '')
 for comp in top_prodCountries:
test['production_country_' + comp] = test['production_countries_collapsed'].\
apply(lambda x: 1 if comp in x else 0)
 test['cast_collapsed'] = test['cast'].apply(lambda x: ' '.\
join(sorted([i['name'] for i in x])) if x != {} else '')
 for lead in top_lead:
test['cast_' + lead] = test['cast_collapsed'].apply(lambda x: 1 if lead in x else 0)
 test['directors_collapsed'] = test['crew'].apply(lambda x: ' '.\
join(sorted([i['name'] for i in x if i['job'] == 'Director'])) if x != {} else '')
 for d in top_directors:
test['director_' + d] = test['directors_collapsed'].apply(lambda x: 1 if d in x else 0)
 for k in keywords_new:
test['txt_' + k] = test['text_merged'].apply(lambda x: 1 if k in x else 0 )
dir_paths = {'base_dir': test_file_dir, 'base_t_dir': root_dir, 'test_file':test_file,
'test_file_translated1': test_eng_file1, 'test_file_translated2': test_eng_file2}
_, _, test, _ = utility.read_data(dir_paths, list(LANGS.keys()), english_pipeline=english_pipeline)
 if english_pipeline:
test_yandex, test_google = test[0], test[1]
test = test_yandex.append(test_google)
 input_cols_test = ['content']
df_test = clean.clean_data(test, input_cols_test, True)
 test.head()<load_pretrained>
cols2drop = ['genres', 'production_companies', 'production_countries', 'genres_collapsed',
'production_companies_collapsed', 'production_countries_collapsed', 'cast_collapsed',
'directors_collapsed', 'n_cast', 'n_crew']
train = train.drop(cols2drop, axis=1)
 test = test.drop(cols2drop, axis=1 )
tokenizer = AutoTokenizer.from_pretrained(str(model_dir))<create_dataframe>
def RMSE(y_obs, y_pred):
n = len(y_obs)
 rmse = np.sqrt(1/n*np.sum(( y_pred-y_obs)**2))
return rmse
def RMSLE(y_obs, y_pred):
n = len(y_obs)
 rmsle = np.sqrt(1/n*np.sum(( np.log(y_pred)-np.log(y_obs)) **2))
return rmsle
def LinearRegression(X, y):
intercept_term = np.ones(shape = y.shape)
 X = np.concatenate(( intercept_term, X), 1)
 coeffs = np.linalg.inv(X.T.dot(X)).dot(X.T ).dot(y)
 return coeffs
test_dataset = models.DatasetRetriever(
 tokenizer,
labels_or_ids=test.index.values,
comment_texts=test[input_cols_test].values,
langs=test['lang'].values,
maxlen=MAX_LENGTH,
use_train_transforms=False,
test=True 
)
 del test
gc.collect() ;<load_pretrained>
x = train['budget'].values
X = np.reshape(x,(len(x),1))
y = train['revenue'].values
y = np.reshape(y,(len(y),1))
np.shape(X), np.shape(y )
transformer = AutoModel.from_config(AutoConfig.from_pretrained(str(model_dir)))
 net = models.ToxicSimpleNNModel(transformer, config.TrainGlobalConfig)
 checkpoint = torch.load(checkpoint_dir, map_location=torch.device('cpu'))
net.load_state_dict(checkpoint);
config.TrainGlobalConfig.train_lenght = len(test_dataset)
 checkpoint = None
del checkpoint<define_search_model>
model_baseline_coeffs = LinearRegression(X, y)
 model_baseline_pred_train = model_baseline_coeffs[0] + model_baseline_coeffs[1]*X
model_baseline_coeffs
warnings.filterwarnings("ignore")
 def _mp_fn(rank, flags):
device = xm.xla_device()
net.to(device)
 test_sampler = torch.utils.data.distributed.DistributedSampler(
 test_dataset,
num_replicas=xm.xrt_world_size() ,
rank=xm.get_ordinal() ,
shuffle=False 
)
 test_loader = torch.utils.data.DataLoader(
 test_dataset,
batch_size=16,
sampler=test_sampler,
pin_memory=False,
drop_last=False,
num_workers=1 
)
 fitter = models.TPUFitter(model=net, device=device, config=config.TrainGlobalConfig,
base_model_path=model_dir, model_name=MODEL, model_prefix=MODEL_PREFIX,
model_version=MODEL_VERSION,
out_path=out_dir)
 fitter.run_inference(test_loader )<load_from_csv>
RMSLE(y, model_baseline_pred_train )
submission = pd.concat([pd.read_csv(path)for path in(out_dir/'node_submissions' ).glob('*.csv')] ).groupby('id' ).mean()
submission['toxic'].hist(bins=100 )<save_to_csv>
train_selected = train[['budget', 'popularity', 'runtime',
'release_date_year', 'release_date_weekday', 'release_date_month',
'budget_yn', 'belongs2coll_yn', 'homepage_yn',
'n_genres', 'n_production_companies', 'n_production_countries',
'revenue']+keys_genre+keys_cast+keys_director+keys_production_company+
keys_production_country+keys_language]
train_selected = train_selected.dropna(axis = 0)
 X = train_selected.drop(['revenue'], axis=1)
 y = train_selected['revenue']
submission.to_csv('submission.csv' )<import_modules>
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4)
 model_RF = RandomForestRegressor(n_estimators = 100,
max_depth = 20)
 model_RF.fit(X_train, y_train)
 model_RF_pred_valset = model_RF.predict(X_test)
 RMSLE(y_test, model_RF_pred_valset )
import os
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
from kaggle_datasets import KaggleDatasets
import transformers
from transformers import TFAutoModel, AutoTokenizer
from tqdm.notebook import tqdm
from tokenizers import Tokenizer, models, pre_tokenizers, decoders, processors<categorify>
model_RF.fit(X, y)
 model_RF_pred_train = model_RF.predict(X)
 RMSLE(y, model_RF_pred_train )
def fast_encode(texts, tokenizer, chunk_size=64, maxlen=256):
tokenizer.enable_truncation(max_length=maxlen)
 tokenizer.enable_padding(max_length=maxlen)
 all_ids = []
for i in tqdm(range(0, len(texts), chunk_size)) :
text_chunk = texts[i:i+chunk_size].tolist()
encs = tokenizer.encode_batch(text_chunk)
 all_ids.extend([enc.ids for enc in encs])
 return np.array(all_ids )<categorify>
X_train, X_test, y_train, y_test = train_test_split(X, np.log1p(y), test_size=0.4)
 model_CatBoost = CatBoostRegressor(silent=True)
 model_CatBoost.fit(X_train, y_train)
 model_CatBoost_pred_valset = model_CatBoost.predict(X_test)
 RMSE(y_test, model_CatBoost_pred_valset )
def regular_encode(texts, tokenizer, maxlen=512):
enc_di = tokenizer.batch_encode_plus(
 texts,
return_attention_masks=False,
return_token_type_ids=False,
pad_to_max_length=True,
max_length=maxlen 
)
 return np.array(enc_di['input_ids'] )<choose_model_class>
model_CatBoost.fit(X, np.log1p(y))
model_CatBoost_pred_train = model_CatBoost.predict(X)
 RMSE(np.log1p(y), model_CatBoost_pred_train )
def build_model(transformer, max_len=256):

 input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
out = Dense(1, activation='relu' )(cls_token)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 return model<choose_model_class>
X_train, X_test, y_train, y_test = train_test_split(X, np.log1p(y), test_size=0.4)
 model_gboost = GradientBoostingRegressor()
model_gboost.fit(X_train, y_train)
 model_gboost_pred_valset = model_gboost.predict(X_test)
 RMSE(y_test, model_gboost_pred_valset )
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
<define_variables>
model_gboost.fit(X, np.log1p(y))
model_gboost_pred_train = model_gboost.predict(X)
 RMSE(np.log1p(y), model_gboost_pred_train )
AUTO = tf.data.experimental.AUTOTUNE
GCS_DS_PATH = KaggleDatasets().get_gcs_path()
EPOCHS = 50
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
AUG_BATCH = BATCH_SIZE
MAX_LEN=256
SEED = 999
LR = 0.0005
MODEL = 'jplu/tf-xlm-roberta-large'<load_pretrained>
model_ensemble_pred_train =(model_CatBoost_pred_train+model_gboost_pred_train)/2
RMSE(np.log1p(y), model_ensemble_pred_train )
tokenizer = AutoTokenizer.from_pretrained(MODEL )<load_from_csv>
model_baseline_pred_test = model_baseline_coeffs[0] + model_baseline_coeffs[1]*X
train1 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv")
 train2 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv")
 train2.toxic = train2.toxic.round().astype(int)
 valid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')
 test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv' )<concatenate>
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['revenue'] = model_baseline_pred_test
submission.to_csv('submission_baseline.csv', index = False )
train = pd.concat([
train1[['comment_text', 'toxic']],
train2[['comment_text', 'toxic']].query('toxic==1'),
train2[['comment_text', 'toxic']].query('toxic==0' ).sample(n=100000, random_state=99)
 ] )<prepare_x_and_y>
test_features = test[['budget', 'popularity', 'runtime',
'release_date_year', 'release_date_weekday', 'release_date_month',
'budget_yn', 'belongs2coll_yn', 'homepage_yn',
'n_genres', 'n_production_companies', 'n_production_countries']+
keys_genre+keys_cast+keys_director+keys_production_company+keys_production_country+
keys_language]
X = test_features
test_features.columns[test_features.isna().any() ].tolist()
%%time
x_train = regular_encode(train.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_valid = regular_encode(valid.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_test = regular_encode(test.content.values, tokenizer, maxlen=MAX_LEN)
 y_train = train.toxic.values
y_valid = valid.toxic.values<categorify>
test_features['runtime'][test_features['runtime'].isna() == True]
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle() 
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(BATCH_SIZE)
 )<load_pretrained>
test_features['runtime'][test_features['runtime'].isna() == True] = test_features['runtime'].median()
%%time
with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model(transformer_layer, max_len=MAX_LEN)
 model.summary()<train_model>
model_RF_pred_test = model_RF.predict(X )
n_steps = x_train.shape[0] // BATCH_SIZE
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS
)<train_model>
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['revenue'] = model_RF_pred_test
submission.to_csv('submission_RF.csv', index = False )
n_steps = x_valid.shape[0] // BATCH_SIZE
early_stopping = tf.keras.callbacks.EarlyStopping(monitor = 'val_auc', mode = 'max', patience = 5,
verbose = 1, min_delta = 0.0001, restore_best_weights = True)
 cb_lr_schedule = tf.keras.callbacks.ReduceLROnPlateau(monitor = 'val_auc', factor = 0.4, patience = 2, verbose = 1, min_delta = 0.0001, mode = 'max')
 history = model.fit(train_dataset,
steps_per_epoch = STEPS_PER_EPOCH,
epochs = EPOCHS,
callbacks = [early_stopping, cb_lr_schedule],
validation_data = val_dataset,
verbose = 2)
 train_history_2 = model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
epochs=EPOCHS
)<save_to_csv>
model_CatBoost_pred_test = model_CatBoost.predict(X)
 model_CatBoost_pred_test = np.expm1(model_CatBoost_pred_test )
sub['toxic'] = model.predict(test_dataset, verbose=1)
 sub.to_csv('submission.csv', index=False )<import_modules>
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['revenue'] = model_CatBoost_pred_test
submission.to_csv('submission_CatBoost.csv', index = False )
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
from tqdm.notebook import tqdm
import os 
<load_from_csv>
model_gboost_pred_test = model_gboost.predict(X)
 model_gboost_pred_test = np.expm1(model_gboost_pred_test )
submission1 = pd.read_csv('/kaggle/input/sub0427/submission9404.csv')
 submission9412 = pd.read_csv('/kaggle/input/sub0427/submission9412.csv')
 submission9463 = pd.read_csv('/kaggle/input/haveatry/submission9463.csv')
 submission4 = pd.read_csv('/kaggle/input/sub0427/submissionen9115.csv')
 submission5 = pd.read_csv('/kaggle/input/sub0427/submission9408.csv')
 submission6 = pd.read_csv('/kaggle/input/haveatry/submission9435.csv')
 submission9477 = pd.read_csv('/kaggle/input/haveatry/submission9477.csv')
 submission9438 = pd.read_csv('/kaggle/input/haveatry/submission9438.csv')
 submission_val9710 = pd.read_csv('/kaggle/input/haveatry/submission_val9710.csv')
 submission9480 = pd.read_csv('/kaggle/input/haveatry/submission9480.csv')
 submission9471 = pd.read_csv('/kaggle/input/haveatry/submission9471.csv')
 submission9472 = pd.read_csv('/kaggle/input/haveatry/submission9472.csv')
 submission9460 = pd.read_csv('/kaggle/input/haveatry/submission_train2_9460.csv')
 submission9454 = pd.read_csv('/kaggle/input/haveatry/submission9454.csv')
 submission9457 = pd.read_csv('/kaggle/input/haveatry/submission9457.csv')
 submission9458 = pd.read_csv('/kaggle/input/haveatry/submission9458.csv')
 submission9481 = pd.read_csv('/kaggle/input/haveatry/submission9481.csv')
 submission9482 = pd.read_csv('/kaggle/input/haveatry/submission9482.csv')
 submission9483 = pd.read_csv('/kaggle/input/haveatry/submission9483.csv')
 submission9484 = pd.read_csv('/kaggle/input/haveatry/submission9484.csv')
 submission9486 = pd.read_csv('/kaggle/input/haveatry/submission9486.csv')
 submission9487 = pd.read_csv('/kaggle/input/haveatry/submission9487.csv')
 submission9488 = pd.read_csv('/kaggle/input/haveatry/submission9488.csv')
 submission9442 = pd.read_csv('/kaggle/input/haveatry/submission9442(2 ).csv')
 submission9443 = pd.read_csv('/kaggle/input/haveatry/submission9443.csv')
 submission9450 = pd.read_csv('/kaggle/input/haveatry/submission9450.csv')
 submission9459 = pd.read_csv('/kaggle/input/haveatry/submission9459(2 ).csv')
 submission9435 = pd.read_csv('/kaggle/input/haveatry/submission9435(2 ).csv')
 submission9436 = pd.read_csv('/kaggle/input/haveatry/submission9436(2 ).csv')
 submission9446 = pd.read_csv('/kaggle/input/haveatry/submission_mix_9446.csv')
 submission_tr_9437=pd.read_csv('/kaggle/input/haveatry/submission_tr_9437.csv')
 submission_it=pd.read_csv('/kaggle/input/haveatry/submission_it_0.csv')
 submission_pt=pd.read_csv('/kaggle/input/haveatry/submission_pt_0.csv')
 submission_es=pd.read_csv('/kaggle/input/haveatry/submission_es_0.csv')
 submission_fr=pd.read_csv('/kaggle/input/haveatry/submission_fr_0.csv')
 submission_ru=pd.read_csv('/kaggle/input/haveatry/submission_ru_0.csv')
 submission_es_precise=pd.read_csv('/kaggle/input/haveatry/submission_es_precise.csv')
 sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')
 test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv' )<load_from_csv>
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['revenue'] = model_gboost_pred_test
submission.to_csv('submission_gboost.csv', index = False )
submission_mix1 = pd.read_csv('/kaggle/input/haveatry/submission_mix1.csv')
 submission_mix2 = pd.read_csv('/kaggle/input/haveatry/submission_mix2.csv')
 submission_mix3 = pd.read_csv('/kaggle/input/haveatry/submission_mix3.csv' )<load_from_csv>
model_ensemble_pred_test =(model_CatBoost_pred_test + model_gboost_pred_test)/2
<load_from_csv><EOS> 
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['revenue'] = model_ensemble_pred_test
submission.to_csv('submission_ensemble.csv', index = False )