Kaggle competition: TMDB Box Office Prediction
<SOS> metric: RMSLE Kaggle data source: tmdb-box-office-prediction<categorify>
pd.set_option('max_columns', None)
 %matplotlib inline
plt.style.use('ggplot')
 stop = set(stopwords.words('english'))
warnings.filterwarnings("ignore" )
def roundLabels(example):
label=tf.cast(example['s_toxic'],tf.float32)
 label=tf.cond(label>=0.5,lambda :tf.ones_like(label),lambda :tf.zeros_like(label))
example['s_toxic']=label
return example<categorify>
train=pd.read_csv('.. /input/train.csv',index_col=0)
 test=pd.read_csv('.. /input/test.csv',index_col=0)
 train.head()
def perFileDataset(filename):
tf.print("Retrieving from filename",filename)
 return tf.data.TFRecordDataset(filename ).map(_parse_proto)
 def perFileDatasetShuffled(filename):
return perFileDataset(filename ).shuffle(200000)
 def unsqueezeTargetDimensions(features,labels):
labels=tf.cast(labels,tf.float32)
 return features,tf.stack([tf.math.subtract(tf.constant([1.0]),labels),labels],axis=-1),<split>
!pip install --upgrade pip
!pip install imdbpy
ia=IMDb()
start =time.clock()
def fill_col(df,col,key,fill_missing_type,nested_arr):
indexes=df.index[df[col]!=df[col]].tolist()
for index in indexes:
imdb_id=df.loc[index]['imdb_id']
movie=ia.get_movie(imdb_id[2:])
 if key in movie:
if(nested_arr):
arr=[]
for el in movie[key]:
arr.append(str(el))
df.at[index, col] = ','.join(arr)
 else:
df.at[index, col] = ','.join(movie[key])
 else:
if(fill_missing_type=='mean'):
df.at[index, col] = df[col].mean()
elif(fill_missing_type=='others'):
df.at[index, col] = 'others'
elif(fill_missing_type=='None'):
df.at[index, col] = None
list1=['runtime','original_language','popularity','title','crew']
list2=['genres','spoken_languages']
for i in list1:
print(i)
 fill_col(train,i,i+'s','None',False)
 fill_col(test,i,i+'s','None',False)
 for j in list2:
print(j)
 fill_col(train,j,j,'None',False)
 fill_col(test,j,j,'None',False)
 end = time.clock()
print('Running time: %s Seconds'%(end-start))
def toxic_train_en_dataset(shardIndex=-1):
return tf.data.Dataset.from_tensor_slices(toxic_en_files ).shuffle(2048 ).interleave(perFileDataset, num_parallel_calls=AUTO,cycle_length=12 ).map(smoother())
 def toxic_train_translated_dataset(shardIndex=-1):
return tf.data.Dataset.from_tensor_slices(toxic_translated_files ).shuffle(2048*4 ).interleave(perFileDataset, num_parallel_calls=AUTO,cycle_length=12 ).map(smoother(0.1))<prepare_x_and_y>
test['revenue']=-99
def non_toxic_train_en_dataset(shardIndex=-1):
return tf.data.Dataset.from_tensor_slices(non_toxic_en_files ).shuffle(2048 ).interleave(perFileDataset, num_parallel_calls=AUTO,cycle_length=12 ).map(smoother())
 def non_toxic_train_translated_dataset(shardIndex=-1):
return tf.data.Dataset.from_tensor_slices(non_toxic_translated_files ).shuffle(2048*4 ).interleave(perFileDataset, num_parallel_calls=AUTO,cycle_length=12 ).map(smoother(0.1))<create_dataframe>
train['be_is_na']=train['belongs_to_collection'].apply(lambda x :1 if x is np.nan else 0)
 test['be_is_na']=test['belongs_to_collection'].apply(lambda x :1 if x is np.nan else 0)
 train=train.drop('belongs_to_collection',axis=1)
 test=test.drop('belongs_to_collection',axis=1 )
def toxic_train_nen_dataset(shardIndex=-1):
return tf.data.Dataset.from_tensor_slices(toxic_nen_files ).shuffle(2048 ).interleave(perFileDataset ).filter(validationFoldRemoveFilter(validation_fold_index)).map(smoother() ).shuffle(8000)
<create_dataframe>
train.loc[train.index == 16,'revenue'] = 192864
train.loc[train.index == 90,'budget'] = 30000000
train.loc[train.index== 118,'budget'] = 60000000
train.loc[train.index== 149,'budget'] = 18000000
train.loc[train.index== 313,'revenue'] = 12000000
train.loc[train.index == 451,'revenue'] = 12000000
train.loc[train.index == 464,'budget'] = 20000000
train.loc[train.index == 470,'budget'] = 13000000
train.loc[train.index== 513,'budget'] = 930000
train.loc[train.index == 797,'budget'] = 8000000
train.loc[train.index == 819,'budget'] = 90000000
train.loc[train.index == 850,'budget'] = 90000000
train.loc[train.index == 1007,'budget'] = 2
train.loc[train.index== 1112,'budget'] = 7500000
train.loc[train.index == 1131,'budget'] = 4300000
train.loc[train.index == 1359,'budget'] = 10000000
train.loc[train.index == 1542,'budget'] = 1
train.loc[train.index == 1570,'budget'] = 15800000
train.loc[train.index== 1571,'budget'] = 4000000
train.loc[train.index == 1714,'budget'] = 46000000
train.loc[train.index == 1721,'budget'] = 17500000
train.loc[train.index== 1865,'revenue'] = 25000000
train.loc[train.index == 1885,'budget'] = 12
train.loc[train.index == 2091,'budget'] = 10
train.loc[train.index == 2268,'budget'] = 17500000
train.loc[train.index == 2491,'budget'] = 6
train.loc[train.index == 2602,'budget'] = 31000000
train.loc[train.index == 2612,'budget'] = 15000000
train.loc[train.index == 2696,'budget'] = 10000000
train.loc[train.index == 2801,'budget'] = 10000000
train.loc[train.index == 335,'budget'] = 2
train.loc[train.index == 348,'budget'] = 12
train.loc[train.index == 470,'budget'] = 13000000
train.loc[train.index == 513,'budget'] = 1100000
train.loc[train.index == 640,'budget'] = 6
train.loc[train.index == 696,'budget'] = 1
train.loc[train.index == 797,'budget'] = 8000000
train.loc[train.index == 850,'budget'] = 1500000
train.loc[train.index == 1199,'budget'] = 5
train.loc[train.index == 1282,'budget'] = 9
train.loc[train.index== 1347,'budget'] = 1
train.loc[train.index== 1755,'budget'] = 2
train.loc[train.index == 1801,'budget'] = 5
train.loc[train.index == 1918,'budget'] = 592
train.loc[train.index == 2033,'budget'] = 4
train.loc[train.index == 2118,'budget'] = 344
train.loc[train.index == 2252,'budget'] = 130
train.loc[train.index == 2256,'budget'] = 1
train.loc[train.index == 2696,'budget'] = 10000000
def non_toxic_train_nen_dataset() :
return tf.data.Dataset.from_tensor_slices(non_toxic_nen_files ).shuffle(2048 ).interleave(perFileDataset ).filter(validationFoldRemoveFilter(validation_fold_index)).map(smoother() ).shuffle(8000)
<split>
test.loc[test.index== 3033,'budget'] = 250
test.loc[test.index== 3051,'budget'] = 50
test.loc[test.index == 3084,'budget'] = 337
test.loc[test.index == 3224,'budget'] = 4
test.loc[test.index == 3594,'budget'] = 25
test.loc[test.index == 3619,'budget'] = 500
test.loc[test.index == 3831,'budget'] = 3
test.loc[test.index== 3935,'budget'] = 500
test.loc[test.index == 4049,'budget'] = 995946
test.loc[test.index== 4424,'budget'] = 3
test.loc[test.index == 4460,'budget'] = 8
test.loc[test.index == 4555,'budget'] = 1200000
test.loc[test.index== 4624,'budget'] = 30
test.loc[test.index== 4645,'budget'] = 500
test.loc[test.index == 4709,'budget'] = 450
test.loc[test.index == 4839,'budget'] = 7
test.loc[test.index== 3125,'budget'] = 25
test.loc[test.index== 3142,'budget'] = 1
test.loc[test.index == 3201,'budget'] = 450
test.loc[test.index == 3222,'budget'] = 6
test.loc[test.index== 3545,'budget'] = 38
test.loc[test.index == 3670,'budget'] = 18
test.loc[test.index == 3792,'budget'] = 19
test.loc[test.index == 3881,'budget'] = 7
test.loc[test.index == 3969,'budget'] = 400
test.loc[test.index == 4196,'budget'] = 6
test.loc[test.index == 4221,'budget'] = 11
test.loc[test.index == 4222,'budget'] = 500
test.loc[test.index== 4285,'budget'] = 11
test.loc[test.index == 4319,'budget'] = 1
test.loc[test.index == 4639,'budget'] = 10
test.loc[test.index == 4719,'budget'] = 45
test.loc[test.index == 4822,'budget'] = 22
test.loc[test.index == 4829,'budget'] = 20
test.loc[test.index== 4969,'budget'] = 20
test.loc[test.index== 5021,'budget'] = 40
test.loc[test.index== 5035,'budget'] = 1
test.loc[test.index== 5063,'budget'] = 14
test.loc[test.index == 5119,'budget'] = 2
test.loc[test.index== 5214,'budget'] = 30
test.loc[test.index== 5221,'budget'] = 50
test.loc[test.index== 4903,'budget'] = 15
test.loc[test.index == 4983,'budget'] = 3
test.loc[test.index == 5102,'budget'] = 28
test.loc[test.index== 5217,'budget'] = 75
test.loc[test.index == 5224,'budget'] = 3
test.loc[test.index== 5469,'budget'] = 20
test.loc[test.index == 5840,'budget'] = 1
test.loc[test.index == 5960,'budget'] = 30
test.loc[test.index == 6506,'budget'] = 11
test.loc[test.index== 6553,'budget'] = 280
test.loc[test.index == 6561,'budget'] = 7
test.loc[test.index== 6582,'budget'] = 218
test.loc[test.index == 6638,'budget'] = 5
test.loc[test.index== 6749,'budget'] = 8
test.loc[test.index==6759,'budget'] = 50
test.loc[test.index == 6856,'budget'] = 10
test.loc[test.index== 6858,'budget'] = 100
test.loc[test.index == 6876,'budget'] = 250
test.loc[test.index == 6972,'budget'] = 1
test.loc[test.index== 7079,'budget'] = 8000000
test.loc[test.index == 7150,'budget'] = 118
test.loc[test.index == 6506,'budget'] = 118
test.loc[test.index == 7225,'budget'] = 6
test.loc[test.index == 7231,'budget'] = 85
test.loc[test.index == 5222,'budget'] = 5
test.loc[test.index == 5322,'budget'] = 90
test.loc[test.index == 5350,'budget'] = 70
test.loc[test.index == 5378,'budget'] = 10
test.loc[test.index== 5545,'budget'] = 80
test.loc[test.index == 5810,'budget'] = 8
test.loc[test.index== 5926,'budget'] = 300
test.loc[test.index== 5927,'budget'] = 4
test.loc[test.index== 5986,'budget'] = 1
test.loc[test.index == 6053,'budget'] = 20
test.loc[test.index== 6104,'budget'] = 1
test.loc[test.index == 6130,'budget'] = 30
test.loc[test.index == 6301,'budget'] = 150
test.loc[test.index == 6276,'budget'] = 100
test.loc[test.index == 6473,'budget'] = 100
test.loc[test.index== 6842,'budget'] = 30
def unknown_nen_dataset() :
return tf.data.Dataset.from_tensor_slices(unknown_nen_files ).shuffle(2048 ).interleave(perFileDataset,num_parallel_calls=AUTO)
<create_dataframe>
new_data=pd.concat([train,test],axis=0)
 new_data.head()
def valid_nen_dataset() :
return tf.data.Dataset.from_tensor_slices(valid_files ).interleave(perFileDataset ).filter(validationFoldOnlyFilter(validation_fold_index))<create_dataframe>
new_data.isnull().sum()
def finetune_nen_dataset() :
return tf.data.Dataset.from_tensor_slices(valid_files ).shuffle(2048 ).interleave(perFileDataset ).filter(validationFoldRemoveFilter(validation_fold_index))<groupby>
new_data[new_data['release_date'].isnull() ]
%%time
if interativeEnvironment:
total_by_source=Counter()
total_by_toxic=Counter()
total=0
for i,l in tqdm(enumerate(non_toxic_train_nen_dataset())) :
total+=1
total_by_source.update([l["source"].numpy() ])
 total_by_toxic.update([l["s_toxic"].numpy() ])
 total_nen_train_non_toxic=total
total,total_by_toxic.most_common() ,total_by_source.most_common()
else:
total_nen_train_non_toxic=5956<groupby>
new_data['release_date']=new_data['release_date'].fillna('3/20/01' )
%%time
if interativeEnvironment:
total_by_source=Counter()
total_by_toxic=Counter()
total=0
for i,l in tqdm(enumerate(toxic_train_nen_dataset())) :
total+=1
total_by_source.update([l["source"].numpy() ])
 total_by_toxic.update([l["s_toxic"].numpy() ])
 total_nen_train_toxic=total
total,total_by_toxic.most_common() ,total_by_source.most_common()
else:
total_nen_train_toxic=1061<groupby>
new_data['release_year']=pd.to_datetime(new_data['release_date'] ).dt.year
new_data['release_month']=pd.to_datetime(new_data['release_date'] ).dt.month
new_data['release_day']=pd.to_datetime(new_data['release_date'] ).dt.day
new_data['release_dayofweek'] = pd.to_datetime(new_data['release_date'] ).dt.dayofweek
new_data['release_quarter'] = pd.to_datetime(new_data['release_date'] ).dt.quarter
%%time
if interativeEnvironment:
total_by_source=Counter()
total_by_toxic=Counter()
total=0
for i,l in tqdm(enumerate(unknown_nen_dataset())) :
total+=1
total_by_source.update([l["source"].numpy() ])
 total_by_toxic.update([l["s_toxic"].numpy() ])
 total_nen_unknown=total
total,total_by_toxic.most_common() ,total_by_source.most_common()
else:
total_nen_unknown=71078<define_variables>
new_data['release_year'].loc[new_data['release_year']>=2018]-=100
n_steps = NEXAMPLESPEREPOCH // BATCH_SIZE
num_nen=0
if use_validation_during_pretraining:
num_nen+=total_nen_train_toxic+total_nen_train_non_toxic
if use_dann:
num_nen += total_nen_unknown
n_steps,num_nen<categorify>
new_data=new_data.drop('release_date',axis=1 )
def applyOptions(ds):
if use_label_filtering:
ds=ds.filter(_filter_non_confident)
 if use_label_rounding:
ds=ds.map(roundLabels)
 ds=ds.map(smoother())
 return ds
num_dataset_section=2
if use_translated_data:
num_dataset_section+=2
def f0() :
return unknown_nen_dataset().repeat().take(total_nen_unknown)
 def f1() :
return applyOptions(toxic_train_en_dataset().repeat().take(int(( NEXAMPLESPEREPOCH-num_nen)/num_dataset_section)))
 def f2() :
return applyOptions(non_toxic_train_en_dataset().repeat().take(int(( NEXAMPLESPEREPOCH-num_nen)/num_dataset_section)))
 def f3() :
return applyOptions(toxic_train_translated_dataset().repeat().take(int(( NEXAMPLESPEREPOCH-num_nen)/num_dataset_section)))
 def f4() :
return applyOptions(non_toxic_train_translated_dataset().repeat().take(int(( NEXAMPLESPEREPOCH-num_nen)/num_dataset_section)))
 def f5() :
return toxic_train_nen_dataset()
def f6() :
return non_toxic_train_nen_dataset()
branches={0:f0,1:f1,2:f2,3:f3,4:f4,5:f5,6:f6}
mixedDatasetParams=[1,2]
if use_dann:
mixedDatasetParams.append(0)
 if use_validation_during_pretraining:
mixedDatasetParams.extend([5,6])
 if use_translated_data:
mixedDatasetParams.extend([3,4])
 def mapParamsToDataset(params):
return tf.switch_case(params,branches)
<prepare_x_and_y>
weiji=[1929,1930,1930,1931,1932,1933,1934,1935,1936,1937,1938,1939,1940,1941,1942,1943,1944,1945,1950,1951,1952,1953,1961,1962,1963,1964,1965,1966,1967,1968,1969,1971,1973,1974,1975,1980,1981,1982,1983,1984,1985,1986,1987,1988,1989,
1990,1991,1992,1993,1994,1995,1996,1997,1998,1999,2007,2008,2009,2010,2011]
cycle_length=len(mixedDatasetParams)
 def non_parallel_dataset(params=mixedDatasetParams):
print(params)
 return tf.data.Dataset.from_tensor_slices(params ).interleave(mapParamsToDataset, num_parallel_calls=AUTO,cycle_length=cycle_length ).take(NEXAMPLESPEREPOCH)
 if interativeEnvironment:
train_dict_dataset=non_parallel_dataset()
train_dict_dataset<create_dataframe>
for i in weiji:
new_data['is_'+str(i)]=new_data['release_year'].apply(lambda x:1 if x==i else 0 )
non_parallel_dataset()<define_search_space>
new_data['homepage_fact']=new_data['homepage'].apply(lambda x: 0 if x is np.nan else 1 )
a=[1,2,3,4]
b=[1,2,3,4]
a==b<init_hyperparams>
new_data['homepage_end']=new_data[new_data['homepage'].notna() ]['homepage'].str.findall(r'\.( [a-z]+ )(?:\/|$)' ).apply(lambda x:x[0])
 new_data['homepage_end'].head()
%%time
if interativeEnvironment:
dlist=non_parallel_dataset().shuffle(NEXAMPLESPEREPOCH ).repeat()
epoch_metrics={}
shard_metrics={}
signature_examples_source={}
signature_examples_count={}
for i,l in tqdm(enumerate(dlist)) :
epoch=str(i//NEXAMPLESPEREPOCH)
 shard=str(i//40000)
 if epoch=="20":
break
if epoch in epoch_metrics.keys() :
pass
else:
total_by_source=Counter()
total_by_toxic=Counter()
total_by_toxic_source=Counter()
total_by_s_toxic=Counter()
total=0
epoch_metrics[epoch]={"by_toxic":total_by_toxic,"by_source":total_by_source,"total":total,"by_both":total_by_toxic_source,"by_s_toxic":total_by_s_toxic}
if shard in shard_metrics.keys() :
pass
else:
stotal_by_source=Counter()
stotal_by_toxic=Counter()
stotal_by_toxic_source=Counter()
stotal_by_s_toxic=Counter()
stotal=0
shard_metrics[shard]={"by_toxic":stotal_by_toxic,"by_source":stotal_by_source,"total":stotal,"by_both":stotal_by_toxic_source,"by_s_toxic":stotal_by_s_toxic}
total+=1
if l["source"].numpy() not in signature_examples_source:
signature_examples_source[l["source"].numpy() ]=l
signature_examples_count[l["source"].numpy() ]=0
if all(signature_examples_source[l["source"].numpy() ]["tokens"].numpy() ==l["tokens"].numpy()):
signature_examples_count[l["source"].numpy() ]+=1
total_by_source.update([l["source"].numpy() ])
 total_by_toxic.update([-1 if l["toxic"] ==-1 else 1 if l["toxic"].numpy() >=0.5 else 0])
 total_by_s_toxic.update([-1 if l["s_toxic"] ==-1 else 1 if l["s_toxic"].numpy() >=0.5 else 0])
 total_by_toxic_source.update([(round(l["toxic"].numpy()),l["source"].numpy())])
 stotal+=1
stotal_by_source.update([l["source"].numpy() ])
 stotal_by_toxic.update([-1 if l["toxic"] ==-1 else 1 if l["toxic"].numpy() >=0.5 else 0])
 stotal_by_s_toxic.update([-1 if l["s_toxic"] ==-1 else 1 if l["s_toxic"].numpy() >=0.5 else 0])
 stotal_by_toxic_source.update([(round(l["toxic"].numpy()),l["source"].numpy())])
 for epoch,epoch_metric in epoch_metrics.items() :
print("Epoch:-",epoch,"Toxic:-",epoch_metric["by_toxic"].most_common() ,"By SToxic:",epoch_metric["by_s_toxic"].most_common() ,"By source:-",epoch_metric["by_source"].most_common() ,"
")
 print(signature_examples_count )<init_hyperparams>
new_data['homepage_end']=new_data['homepage_end'].fillna('unknow' )
if interativeEnvironment:
for epoch,epoch_metric in shard_metrics.items() :
print("Shard:-",epoch,"Toxic:-",epoch_metric["by_toxic"].most_common() ,"Stoxic:",epoch_metric["by_s_toxic"].most_common() ,"By source:-",epoch_metric["by_source"].most_common() ,"
" )<load_pretrained>
page=pd.get_dummies(new_data['homepage_end'])
 page.head()
def opus_files(lang):
return tf.io.gfile.glob(OPUS_DS_PATH + "/{}_*.tfrecord".format(lang)+extension)
 opus_files("es" )<data_type_conversions>
new_data=new_data.drop('homepage',axis=1 )
if use_parcor:
parallel_feature_description = {
'tokens_en': tf.io.FixedLenFeature([192], tf.int64),
'en_attn_mask': tf.io.FixedLenFeature([192], tf.int64),
'tokens_nen': tf.io.FixedLenFeature([192], tf.int64),
'nen_attn_mask': tf.io.FixedLenFeature([192], tf.int64),
'lang':tf.io.FixedLenFeature([], tf.string),
'source':tf.io.FixedLenFeature([], tf.int64,default_value=[5]),
's_toxic':tf.io.FixedLenFeature([2], tf.float32,default_value=[-1.0,-1.0]),
'is_same':tf.io.FixedLenFeature([], tf.float32,default_value=[1.0]),
}
def _parse_parallel_proto(example_proto):
example= tf.io.parse_single_example(example_proto, parallel_feature_description)
 return example
if IN_COLAB:
compression_type="GZIP"
else:
compression_type=""
def perFileParallelDataset(filename,feature_description=feature_description):
tf.print("Retrieving from filename",filename)
 return tf.data.TFRecordDataset(filename,compression_type=compression_type ).map(_parse_parallel_proto)
 def make_parallel_pairs(example):
ret={}
ret["tokens"]=tf.stack([example["tokens_en"],example["tokens_nen"]])
 ret["mask"]=tf.stack([example["en_attn_mask"],example["nen_attn_mask"]])
 ret["s_toxic"]=example["s_toxic"]
ret["is_same"]=example["is_same"]
ret["lang"]=example["lang"]
return ret
def parallel_dataset(files):
return tf.data.Dataset.from_tensor_slices(files ).shuffle(2048 ).interleave(perFileParallelDataset ).map(make_parallel_pairs)
 def parallel_labelled_dataset(labelled_dataset):
return labelled_dataset.batch(2 ).map(make_parallel_labelled_pairs)
 def make_parallel_labelled_pairs(example):
example["is_same"]=tf.constant(0.0)
 return example
langs=["es","fr","pt","tr","ru","it"]
langFiles=list(map(opus_files,langs))
def all_parallel_dataset() :
return tf.data.Dataset.from_tensor_slices(langFiles ).interleave(parallel_dataset, num_parallel_calls=AUTO,cycle_length=len(langs))
def parallel_nonparallel_mixer(params):
return tf.cond(params=="parallel",lambda :all_parallel_dataset().take(int(NEXAMPLESPEREPOCH/4)).map(_form_tuple),lambda :parallel_labelled_dataset(non_parallel_dataset() ).take(int(NEXAMPLESPEREPOCH/4)).map(_form_tuple))
def parcor_dataset() :
return tf.data.Dataset.from_tensor_slices(["parallel","nonparallel"] ).interleave(parallel_nonparallel_mixer,num_parallel_calls=2 )<concatenate>
new_data=new_data.drop('poster_path',axis=1 )
if interativeEnvironment:
ds=list(parallel_labelled_dataset(valid_nen_dataset() ).take(7))
print(list(map(lambda a:a["is_same"],ds)))
<define_variables>
new_data['len_overview']=new_data['overview'].fillna('NAN' ).apply(lambda x:len(x))
new_data['len_overview'][new_data['len_overview']==0]=1
new_data['len_overview_budget']=new_data['budget']/(new_data['len_overview']+1 )
valid_en_files=list(filter(lambda file:"valid" in file and "nen" not in file and "Sentences" not in file, data_files))
valid_en_files<prepare_x_and_y>
len_rew_sort=new_data['len_overview'].sort_values(ascending=True)
 len_rew_sort.head()
def valid_en_dataset() :
return tf.data.Dataset.from_tensor_slices(valid_en_files ).interleave(perFileDataset ).shuffle(200000 ).map(roundLabels )<categorify>
length=len(new_data['len_overview'])
 m=0.1
n=0.1
arr_len_ove=[]
for i in range(1,11):
arr_len_ove.append(round(length*m))
m+=n
arr_len_ove
if use_parcor:
train_dataset=parcor_dataset().shuffle(NEXAMPLESPEREPOCH ).repeat().batch(int(BATCH_SIZE/2)).prefetch(AUTO)
 valid_dataset=parallel_labelled_dataset(valid_nen_dataset() ).map(_form_tuple ).batch(int(BATCH_SIZE/2)).prefetch(AUTO)
 finetune_dataset=parallel_labelled_dataset(finetune_nen_dataset() ).map(_form_tuple ).batch(int(BATCH_SIZE/2)).prefetch(AUTO)
 else:
if use_english_validation:
valid_dataset = valid_en_dataset().batch(BATCH_SIZE ).map(_form_tuple ).prefetch(AUTO)
 else:
valid_dataset = valid_nen_dataset().map(_form_tuple ).batch(BATCH_SIZE ).prefetch(AUTO)
 train_dataset=non_parallel_dataset().shuffle(NEXAMPLESPEREPOCH ).map(_form_tuple ).repeat().batch(BATCH_SIZE ).prefetch(AUTO)
 finetune_dataset=finetune_nen_dataset().shuffle(NEXAMPLESPEREPOCH ).map(_form_tuple ).batch(BATCH_SIZE ).prefetch(AUTO)
 valid_dataset<import_modules>
for i in range(10):
qu=qu_arr[i]
if i==0:
new_data['len_overview'].loc[(new_data['len_overview']<len_rew_sort.iloc[arr_len_ove[i]-1])]=qu
else:
new_data['len_overview'].loc[(new_data['len_overview']<len_rew_sort.iloc[arr_len_ove[i]-1])&(new_data['len_overview'] >qu_arr[i-1])]=qu
print(i,qu )
class WeightedBinaryCrossentropy(tf.keras.losses.BinaryCrossentropy):
def __init__(self,
from_logits=False,
label_smoothing=0,
reduction=losses_utils.ReductionV2.AUTO,lossWeight=0.1,
name='binary_crossentropy'):

 super(WeightedBinaryCrossentropy, self ).__init__(
 name=name,
reduction=reduction,
from_logits=from_logits,
label_smoothing=label_smoothing)
 self.lossWeight=lossWeight
def call(self, y_true, y_pred):
l= tf.math.scalar_mul(self.lossWeight,super().call(y_true,y_pred))
return l<import_modules>
len_ove_agg=new_data.groupby('len_overview' ).revenue.aggregate(['min','max','std'] )
class MaskedBinaryCrossentropy(tf.keras.losses.BinaryCrossentropy):
def __init__(self,
from_logits=False,
label_smoothing=0,
reduction=losses_utils.ReductionV2.AUTO,
name='binary_crossentropy'):

 super(MaskedBinaryCrossentropy, self ).__init__(
 name=name,
reduction=reduction,
from_logits=from_logits,
label_smoothing=label_smoothing)
 def call(self, y_true, y_pred):
if use_parcor:
raise("Unimplemented")
 y_true=tf.reshape(y_true,[-1,1])
 y_pred=tf.reshape(y_pred,[-1,1])
 mask=tf.math.greater_equal(y_true,0)
 mask=tf.cast(mask,y_true.dtype)
 mask=tf.squeeze(mask,axis =-1)
 masked_y_true=mask * y_true
l=super().call(masked_y_true,y_pred)
 l=mask*l
if use_parcor:
l=tf.reshape(l,[-1,2])
 l=l[:,0]+l[:,1]
return l<compute_test_metric>
new_data['geres_name']=new_data['genres'].str.findall(r''name'\s?:\s?'(\w+)'')
 new_data['geres_name'].head()
class SquareLoss(tf.keras.losses.Loss):
def __init__(self,reduction=losses_utils.ReductionV2.AUTO, name="squareLoss",lossWeight=0):
super().__init__(reduction=reduction,name=name)
 self.lossWeight=lossWeight
def call(self, y_true, y_pred):
is_same_masked= y_pred*y_true
loss=tf.math.scalar_mul(self.lossWeight,is_same_masked*is_same_masked)
 return loss<categorify>
new_data=new_data.drop('genres',axis=1 )
def build_model(transformer, max_len=512):
if use_parcor:
pair_input_word_ids = Input(shape=(2,max_len,), dtype=tf.int32, name="input_word_ids")
 pair_attention_masks = Input(shape=(2,max_len,), dtype=tf.int32, name="attention_mask")
 input_word_ids = tf.keras.backend.reshape(pair_input_word_ids,shape=(-1,max_len))
attention_masks= tf.keras.backend.reshape(pair_attention_masks,shape=(-1,max_len))
inputs=[pair_input_word_ids,pair_attention_masks]
else:
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 attention_masks = Input(shape=(max_len,), dtype=tf.int32, name="attention_mask")
 inputs=[input_word_ids,attention_masks]
transformer_output = transformer(( input_word_ids,attention_masks))
sequence_output = transformer_output[0]
transformer_output
if pooling_mode == pooling_mode_cls_token:
transformed_features = sequence_output[:, 0, :]
cls_token=transformed_features
else:
float_mask=tf.cast(tf.expand_dims(attention_masks,axis=-1),tf.float32)
 lengths=tf.reduce_sum(float_mask,axis=-2)
 masked_vals=tf.multiply(sequence_output,float_mask)
 sums=tf.reduce_sum(masked_vals,axis=-2)
 avgs=tf.math.divide(sums,lengths)
 maxes=tf.math.reduce_max(masked_vals,axis=-2)
 transformed_features=tf.concat([avgs,maxes],axis=-1)
 cls_token=transformed_features
for dims in fc_dims:
cls_token=Dense(dims, activation='relu' )(cls_token)
 if use_dann:
lang_token=GradReverse()(transformed_features)
 for dims in fc_dims:
lang_token=Dense(dims, activation='relu' )(lang_token)
 lang=Dense(1, activation="sigmoid",name="lang" )(lang_token)
 if use_parcor:
toxic = Dense(1, activation="sigmoid",name="unpair_toxic" )(cls_token)
 toxic=tf.keras.backend.reshape(toxic,shape=(-1,2,1))
toxic=tf.keras.layers.Layer(name="toxic" )(toxic)
 is_same=tf.keras.layers.Subtract(name="is_same" )([toxic[:,1,:],toxic[:,0,:]])
 else:
toxic = Dense(1, activation="sigmoid",name="toxic" )(cls_token)
 out=[toxic]
if use_parcor:
out.append(is_same)
 celoss=MaskedBinaryCrossentropy(
 from_logits=False, label_smoothing=0, reduction=losses_utils.ReductionV2.AUTO,
name='crossentropy_loss' 
)
 loss={"toxic":celoss}
if use_dann:
domainGap=WeightedBinaryCrossentropy(
 from_logits=False, label_smoothing=0, reduction=losses_utils.ReductionV2.AUTO,
name='domaingap_loss',lossWeight=dann_lambda 
)
 out.append(lang)
 loss["lang"]=domainGap
if use_parcor:
loss["is_same"]=SquareLoss(lossWeight=parcor_lambda)
 model = Model(inputs=inputs, outputs=out)
 model.compile(Adam(lr=5e-6), loss=loss, metrics={"toxic":['accuracy',tf.keras.metrics.AUC(name="auc"),tf.keras.metrics.AUC(name="pr-auc",curve="PR")]})
 return model<categorify>
country=new_data['production_countries'].str.findall(r'[A-Z]{2,5}')
def setTrainableLayers(transformer,embeddingsTrainable=False,poolerTrainable=False,hiddenTrainable=8):
transformer.layers[0].pooler.trainable=poolerTrainable
transformer.layers[0].embeddings.trainable=embeddingsTrainable
trainableHiddens=transformer.layers[0].encoder.layer[-1*hiddenTrainable:]
for layer in transformer.layers[0].encoder.layer:
if layer in trainableHiddens:
layer.trainable=True
else:
layer.trainable=False<load_pretrained>
new_data['production_countries']=country
%%time
with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model(transformer_layer, max_len=MAX_LEN)
 if load_model is not None:
model.load_weights(load_model)
 model.summary()<define_variables>
new_data['original_and_new']=new_data['original_title']==new_data['title']
now = datetime.now()
dt_string = now.strftime("%d_%m_%Y_%H_%M_%S_")
 out_path=dt_string+'zero_shot_pro.h5'
write_root=""
if IN_COLAB:
write_root=root+"outputs/"
out_path=write_root+out_path
print(out_path)
<choose_model_class>
new_data=new_data.drop('original_title',axis=1)
 new_data.head()
if use_pretraining:
metric_prefix=""
if use_parcor or use_dann:
metric_prefix="_toxic"
mc = ModelCheckpoint(out_path, monitor="val"+metric_prefix+"_auc", mode='max',verbose=1,save_weights_only=True,save_best_only=True)
 es = EarlyStopping(monitor="val"+metric_prefix+"_auc", mode='max',patience=2,verbose=1,restore_best_weights=True)
 lrScheduler=ReduceLROnPlateau(
 monitor="val"+metric_prefix+"_loss",
factor=0.2,
patience=1,
verbose=1,
mode="min",
min_delta=0,
cooldown=0,
min_lr=1e-10,
)<train_model>
new_data=pd.concat([new_data,page],axis=1 )
if use_pretraining:
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS,
verbose=1,
callbacks=[mc,es,lrScheduler]
)<load_from_csv>
new_data['production_companies']=new_data['production_companies'].str.findall(r''name'?:\s?'([A-Za-z]+)')
 new_data.fillna('Unknow')
 print('接下来就是地图可视化了' )
%%time
sub = pd.read_csv(root+'/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv',nrows=nrows )<define_variables>
new_data=new_data.drop('imdb_id',axis=1 )
test_files=list(filter(lambda file:"test" in file and "nen" in file, data_files))
test_files<categorify>
new_data['spoken_languages']=new_data['spoken_languages'].str.findall(r''([a-z]{2})'' )
if use_parcor:
test_dataset=parallel_labelled_dataset(tf.data.Dataset.from_tensor_slices(test_files ).interleave(perFileDataset,cycle_length=1)).map(_form_tuple ).batch(int(BATCH_SIZE/2)).prefetch(AUTO)
 else:
test_dataset=tf.data.Dataset.from_tensor_slices(test_files ).interleave(perFileDataset,cycle_length=1 ).batch(BATCH_SIZE ).map(_form_tuple ).prefetch(AUTO)
<save_to_csv>
new_data['production_companies']=new_data['production_companies'].fillna('unknow' )
if use_pretraining:
l=model.predict(test_dataset, verbose=1)
 if use_dann:
l=l[0]
if use_parcor:
l=np.reshape(l,(-1,1))
sub['toxic'] = l[:,-1]
if use_finetuning:
write_file=write_root+'submission.unfinetuned.csv'
else:
write_file=write_root+'submission.csv'
sub.to_csv(write_file, index=False )<define_variables>
new_data['spoken_languages']=new_data['spoken_languages'].fillna('unknow' )
if use_finetuning:
now = datetime.now()
dt_string = now.strftime("%d_%m_%Y_%H_%M_%S_")
 out_path=dt_string+'zero_shot_pro.h5'
write_root=""
if IN_COLAB:
write_root=root+"outputs/"
out_path=write_root+out_path
print(out_path)
<choose_model_class>
new_data['Keywords']=new_data['Keywords'].str.findall(r''?:\s?'([a-z]+\s?[a-z]+)'' ).fillna('unkonw' )
if use_finetuning:
metric_prefix=""
if use_parcor or use_dann:
metric_prefix="_toxic"
mc = ModelCheckpoint(out_path, monitor="val"+metric_prefix+"_auc", mode='max',verbose=1,save_weights_only=True,save_best_only=True)
 es = EarlyStopping(monitor="val"+metric_prefix+"_auc", mode='max',patience=2,verbose=1,restore_best_weights=True)
 lrScheduler=ReduceLROnPlateau(
 monitor="val"+metric_prefix+"_loss",
factor=0.2,
patience=1,
verbose=1,
mode="min",
min_delta=0,
cooldown=0,
min_lr=1e-10,
)<train_model>
d = Counter([i for j in list_of_keywords for i in j] ).most_common(30)
 d
if use_finetuning:
train_history = model.fit(
 finetune_dataset,
validation_data=valid_dataset,
epochs=EPOCHS,
verbose=1,
callbacks=[mc,es,lrScheduler]
)<save_to_csv>
for i in d:
m=i[0]
new_data['Keywords_'+m]=new_data['Keywords'].apply(lambda x:1 if m in x else 0 )
if use_finetuning:
l=model.predict(test_dataset, verbose=1)
 if use_dann:
l=l[0]
if use_parcor:
l=np.reshape(l,(-1,1))
sub['toxic'] = l[:,-1]
write_file=write_root+'submission.csv'
sub.to_csv(write_file, index=False )<set_options>
new_data=new_data.drop('Keywords',axis=1 )
def reset_keras() :
tf.keras.backend.clear_session()
print(gc.collect() )<install_modules>
new_data['cast']=new_data['cast'].fillna('unknow' )
!pip install -q nltk
!pip install -q WordCloud
!pip install -q plotly
!pip install -q transformers
!pip install -q twython<import_modules>
for i in d:
m=i[0][0]
new_data['cast_name_'+m]=new_data['cast'].apply(lambda x:1 if m in x else 0 )
nltk.download('stopwords')
 nltk.download('punkt')
 nltk.download('vader_lexicon')
<load_from_csv>
list_cast_gender=list(new_data['cast'].str.findall(r''gender'\s?:\s?(\d+)\s?'))
Counter([i for j in list_cast_gender for i in j] ).most_common()
dir_path = '/kaggle/input/jigsaw-multilingual-toxic-comment-classification/'
train_data_1 = pd.read_csv(dir_path + "/jigsaw-toxic-comment-train.csv")
 train_data_2 = pd.read_csv(dir_path + "/jigsaw-unintended-bias-train.csv")
 validation_data = pd.read_csv(dir_path + "/validation.csv")
 test_data = pd.read_csv(dir_path + "/test.csv" )<categorify>
new_data['cast_gender_0']=new_data['cast'].str.findall(r''gender'\s?:\s?(\d+)\s?' ).apply(lambda x: x.count('0'))
new_data['cast_gender_1']=new_data['cast'].str.findall(r''gender'\s?:\s?(\d+)\s?' ).apply(lambda x: x.count('1'))
new_data['cast_gender_2']=new_data['cast'].str.findall(r''gender'\s?:\s?(\d+)\s?' ).apply(lambda x: x.count('2'))
new_data['budget_gender_count']=new_data['budget']/(new_data['cast_gender_0']+new_data['cast_gender_1']+new_data['cast_gender_2']+1 )
train_data_2.toxic = train_data_2.toxic.round().astype(int)
 train_data = pd.concat([
train_data_1[['comment_text','toxic']]
, train_data_2[['comment_text','toxic']].query('toxic == 1')
 , train_data_2[['comment_text', 'toxic']].query('toxic == 0' ).sample(n = 200000, random_state = 1993)
 ])
<string_transform>
for g in top_cast_char:
m=g[0]
new_data['cast_char_'+m]=new_data['cast'].apply(lambda x:1 if m in x else 0 )
word_tokenize(train_data.comment_text[1])
<string_transform>
new_data=new_data.drop('cast',axis=1 )
text_words = word_tokenize(train_data.comment_text[1])
 text_words = [word.lower() for word in text_words]<categorify>
new_data['status']=new_data['status'].fillna('Unknow' )
modified_stopwords = stopwords.words('english')
 modified_stopwords.remove('not')
 text_words = [word for word in text_words if not word in modified_stopwords and word.isalpha() ]
len(text_words )<string_transform>
new_data['overview']=new_data['overview'].fillna('' )
def nan_filter(x):
if type(x)== str:
return(x.replace("
", "")).lower()
else:
return ""
nontoxic_text = ' '.join([nan_filter(comment)for comment in train_sample.query('toxic==0')['comment_text']])
 toxic_text = ' '.join([nan_filter(comment)for comment in train_sample.query('toxic == 1')['comment_text']] )<feature_engineering>
from sklearn.linear_model import LinearRegression
import eli5
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
def text_len(x):
if type(x)is str:
return len(x.split())
 else:
return 0
train_sample['comment_size'] = train_sample.comment_text.apply(text_len)
 toxic_text_lengths = train_sample.query('toxic == 1 and comment_size < 200')['comment_size'].sample(frac = 1, random_state = 1993)
 nontoxic_text_lengths = train_sample.query('toxic == 0 and comment_size < 200')['comment_size'].sample(frac = 1, random_state = 1993)
<feature_engineering>
log_re=np.log1p(train['revenue'])
 v1=TfidfVectorizer(sublinear_tf=True,analyzer='word',
token_pattern=r'\w{1,}',
ngram_range=(1, 2),
min_df=5)
 overview_text = v1.fit_transform(train['overview'].fillna(''))
linre=LinearRegression()
linre.fit(overview_text,log_re)
def sentiment(x):
if type(x)is str:
return SIA.polarity_scores(x)
 else:
return 1000
SIA = SentimentIntensityAnalyzer()
train_sample['polarity'] = train_sample.comment_text.apply(sentiment)
<feature_engineering>
eli5.show_weights(linre, vec=v1, top=20, feature_filter=lambda x: x !='<BIAS>' )
train_sample['negativity'] = train_sample.polarity.apply(lambda x: x['neg'])
 train_sample['positivity'] = train_sample.polarity.apply(lambda x: x['pos'] )<define_variables>
new_data['crew_0']=new_data['crew'].fillna('' ).str.replace(',','' ).str.replace('}','' ).str.findall(''gender\S?'\s?:\s?\S?(\d+)\s?\S?' ).apply(lambda x:x.count('0'))
new_data['crew_1']=new_data['crew'].fillna('' ).str.replace(',','' ).str.replace('}','' ).str.findall(''gender\S?'\s?:\s?\S?(\d+)\s?\S?' ).apply(lambda x:x.count('1'))
new_data['crew_2']=new_data['crew'].fillna('' ).str.replace(',','' ).str.replace('}','' ).str.findall(''gender\S?'\s?:\s?\S?(\d+)\s?\S?' ).apply(lambda x:x.count('2'))
roberta_string = 'jplu/tf-xlm-roberta-large'<load_pretrained>
for i in d:
new_data['crew_depart_is_'+i[0]]=new_data['crew'].fillna('' ).apply(lambda x:1 if i[0] in x else 0)
 for i in dd:
new_data['crew_job_is_'+i[0]]=new_data['crew'].fillna('' ).apply(lambda x:1 if i[0] in x else 0 )
tokenizer = AutoTokenizer.from_pretrained(roberta_string )<categorify>
new_data['crew']=new_data['crew'].str.replace(',','' ).str.replace('}','' ).str.replace(r''gender'\s?\S?:\s?\S?\s+'id'\s?:\s?\d+','' ).str.findall(r''department'\s?\S?:\s+\S?(\S+)\S?\s?\S?\s+'job\S?'\s?:\s?'(\D+)\S\s?\S?'name\S?'\s?:\s?'(\D+)'\s?\S?'profile_path\S?':\s+'?(\S+)\S?\s?\s?\S?' )
sample_encoded = tokenizer.batch_encode_plus(train_sample.comment_text.values[0:2]
, return_attention_masks=False
, return_token_type_ids=False
, pad_to_max_length=True
, max_length = 512)
 sample_encoded<categorify>
new_data['len_crew']=new_data['crew'].fillna('1' ).apply(lambda x:len(x))
def encode(text, max_len = 512):
encoded_dict = tokenizer.batch_encode_plus(text
, return_attention_masks=False
, return_token_type_ids=False
, pad_to_max_length=True
, max_length = max_len)
 return np.array(encoded_dict['input_ids'])
 MAX_LEN = 192<set_options>
new_train=new_data.loc[np.array(train.index)]
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<define_variables>
new_train['production_countries']=new_train['production_countries'].fillna('' )
AUTO = tf.data.experimental.AUTOTUNE
GCS_DS_PATH = KaggleDatasets().get_gcs_path()
EPOCHS = 2
BATCH_SIZE = 16 * strategy.num_replicas_in_sync 
<categorify>
new_train['production_countries']=new_train['production_countries'].apply(lambda x:'_'.join(x))
X_train = encode(train_data.comment_text.values, MAX_LEN)
 X_valid = encode(validation_data.comment_text.values, MAX_LEN)
 X_test = encode(test_data.content.values, MAX_LEN)
 y_train = train_data.toxic.values
y_valid = validation_data.toxic.values<define_variables>
count=new_train['production_countries'].value_counts()
count=count[count>5]
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( X_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( X_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(X_test)
.batch(BATCH_SIZE)
 )<choose_model_class>
bud=new_train.groupby('production_countries' ).budget.mean()
%%time
with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(roberta_string)
 input_word_ids = Input(shape=(MAX_LEN,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer_layer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(cls_token)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 model.summary()<train_model>
bud=bud.sort_values(ascending=False)[:10]
n_steps = X_train.shape[0] // BATCH_SIZE
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS
)<train_model>
rev=new_train.groupby('production_countries' ).revenue.mean()
n_steps = X_valid.shape[0] // BATCH_SIZE
train_history_2 = model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
epochs=EPOCHS
)<save_to_csv>
rev=rev.sort_values(ascending=False)[:10]
submission = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')
 submission['toxic'] = model.predict(test_dataset, verbose = 1)
 submission.to_csv('/kaggle/working/submission.csv', index = False )<import_modules>
new_train['production_countries'].loc[new_train['production_countries']=='ET']='Ethiopia'
along_co=new_train[new_train['production_countries'].apply(lambda x:1 if len(x)==2 else 0)==1]
along_co.head()
import os
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
from kaggle_datasets import KaggleDatasets
import transformers
from transformers import TFAutoModel, AutoTokenizer
from tqdm.notebook import tqdm
from tokenizers import Tokenizer, models, pre_tokenizers, decoders, processors
import pickle<choose_model_class>
count_rev=along_co[['production_countries','revenue']]
def build_model(transformer, max_len=512):

 input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(cls_token)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=0.000009), loss='binary_crossentropy', metrics=['accuracy'])
 return model<set_options>
dd=count_rev['production_countries'].unique()
mn=pd.DataFrame(dd,columns=['address'])
 mn.head()
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<define_variables>
import geopandas as ge
AUTO = tf.data.experimental.AUTOTUNE
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
MAX_LEN = 192
MODEL = 'jplu/tf-xlm-roberta-large'<load_pretrained>
world = ge.read_file(ge.datasets.get_path('naturalearth_lowres'))
tokenizer = AutoTokenizer.from_pretrained(MODEL )<load_pretrained>
from mpl_toolkits.axes_grid1 import make_axes_locatable
%%time
filename = "/kaggle/input/jigsawtpuxlmrobertacopypickledata/jigsaw_multilingual_x_train.pkl"
x_train = pickle.load(open(filename, 'rb'))
filename = "/kaggle/input/jigsawtpuxlmrobertacopypickledata/jigsaw_multilingual_x_trainOA.pkl"
x_trainOA = pickle.load(open(filename, 'rb'))
filename = "/kaggle/input/jigsawtpuxlmrobertacopypickledata/jigsaw_multilingual_x_trainA.pkl"
x_trainA = pickle.load(open(filename, 'rb'))
filename = "/kaggle/input/jigsawtpuxlmrobertacopypickledata/jigsaw_multilingual_x_valid.pkl"
x_valid = pickle.load(open(filename, 'rb'))
filename = "/kaggle/input/jigsawtpuxlmrobertacopypickledata/jigsaw_multilingual_x_test.pkl"
x_test = pickle.load(open(filename, 'rb'))
filename = "/kaggle/input/jigsawtpuxlmrobertacopypickledata/jigsaw_multilingual_y_train.pkl"
y_train = pickle.load(open(filename, 'rb'))
filename = "/kaggle/input/jigsawtpuxlmrobertacopypickledata/jigsaw_multilingual_y_trainOA.pkl"
y_trainOA = pickle.load(open(filename, 'rb'))
filename = "/kaggle/input/jigsawtpuxlmrobertacopypickledata/jigsaw_multilingual_y_trainA.pkl"
y_trainA = pickle.load(open(filename, 'rb'))
filename = "/kaggle/input/jigsawtpuxlmrobertacopypickledata/jigsaw_multilingual_y_valid.pkl"
y_valid = pickle.load(open(filename, 'rb'))<split>
dd=count_rev['production_countries'].value_counts()
mn['values']=dd.values
mn.head()
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 train_datasetOA =(
 tf.data.Dataset 
.from_tensor_slices(( x_trainOA, y_trainOA)) 
.repeat() 
.shuffle(x_trainOA.shape[0])
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 train_datasetA =(
 tf.data.Dataset 
.from_tensor_slices(( x_trainA, y_trainA)) 
.repeat() 
.shuffle(x_trainOA.shape[0])
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(BATCH_SIZE)
 )<load_pretrained>
mn['address'][0]='United States'
mn['address'][2]='Korea'
mn['address'][1]='India'
mn['address'][3]='Serbia'
mn['address'][4]='United Kingdom'
mn['address'][5]='France'
mn['address'][6]='New Zealand'
mn['address'][7]='Italy'
mn['address'][8]='Belgium'
mn['address'][9]='Czech Rep.'
mn['address'][11]='Russia'
mn['address'][12]='Spain'
mn['address'][13]='Turkey'
mn['address'][14]='China'
mn['address'][15]='Canada'
mn['address'][16]='Australia'
mn['address'][17]='Iran'
mn['address'][18]='Japan'
mn['address'][19]='Sweden'
mn['address'][20]='Philippines'
mn['address'][21]='Brazil'
mn['address'][22]='Netherlands'
mn['address'][23]='Ireland'
mn['address'][24]='Mexico'
mn['address'][25]='Germany'
mn['address'][26]='South Africa'
mn['address'][27]='Finland'
mn['address'][28]='Pakistan'
mn['address'][29]='Norway'
mn['address'][30]='Bulgaria'
mn['address'][31]='Ukraine'
mn['address'][32]='Denmark'
mn['address'][33]='Romania'
mn['address'][35]='Hungary'
mn['address'][36]='Chile'
mn['address'][37]='Indonesia'
mn['address'][38]='Poland'
%%time
with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model(transformer_layer, max_len=MAX_LEN)
 model.summary()<train_model>
mn=mn.drop(10)
 mn=mn.drop(34 )
n_steps = x_train.shape[0] // BATCH_SIZE
model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=1
)<predict_on_test>
mn['geometry']='unknow'
checkpointPredictions1 = model.predict(test_dataset, verbose=1)
 print(checkpointPredictions1[:10] )<train_model>
for i in range(39):
if i==10 or i==34:
continue;
d=world[world['name']==mn['address'][i]]['geometry'].values[0]
mn['geometry'][i]=d
n_steps = x_trainA.shape[0] // BATCH_SIZE
model.fit(
 train_datasetA,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=1
)<predict_on_test>
TS = train.loc[:,["original_title","release_date","budget","runtime","revenue"]]
TS.dropna()
TS.release_date = pd.to_datetime(TS.release_date)
 TS.loc[:,"Year"] = TS["release_date"].dt.year
TS.loc[:,"Month"] = TS["release_date"].dt.month
TS = TS[TS.Year<2018]
checkpointPredictions2 = model.predict(test_dataset, verbose=1)
 print(checkpointPredictions2[:10] )<train_model>
top3 = train.sort_values(by='popularity',ascending=False)[:10]
id3=top3[['title','poster_path','revenue']]
n_steps = x_valid.shape[0] // BATCH_SIZE
model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
epochs=2
)<predict_on_test>
old_title=new_data['title'].value_counts()
checkpointPredictions3 = model.predict(test_dataset, verbose=1)
 print(checkpointPredictions3[:10] )<train_model>
a=old_title[old_title.values==3].index
b=old_title[old_title==2].index
n_steps = x_trainOA.shape[0] // BATCH_SIZE
model.fit(
 train_datasetOA,
steps_per_epoch=n_steps,
epochs=1
)<predict_on_test>
new_data['fan_pai_2']=new_data['title'].apply(lambda x:1 if x in a else 0)
 new_data['pan_pai_3']=new_data['title'].apply(lambda x:1 if x in b else 0 )
checkpointPredictions4 = model.predict(test_dataset, verbose=1)
 print(checkpointPredictions4[:10] )<save_to_csv>
new_data.loc[new_data['title'].fillna('un' ).str.contains('Planet of the Apes')]
sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')
 sub['toxic'] =(checkpointPredictions1*0.05)+(checkpointPredictions2*0.10)+(checkpointPredictions3*0.76)+(checkpointPredictions4*0.09)
 sub.to_csv('submission.csv', index=False )<import_modules>
top10 = train.sort_values(by='revenue',ascending=False)[:10]
id10=top10[['title','poster_path','revenue']]
import os
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
from kaggle_datasets import KaggleDatasets
import transformers
from transformers import TFAutoModel, AutoTokenizer
from tqdm.notebook import tqdm
from tokenizers import Tokenizer, models, pre_tokenizers, decoders, processors<categorify>
title_count=train['title'].value_counts()
len(title_count[title_count.values!=1] )
def fast_encode(texts, tokenizer, chunk_size=256, maxlen=512):

 tokenizer.enable_truncation(max_length=maxlen)
 tokenizer.enable_padding(max_length=maxlen)
 all_ids = []
for i in tqdm(range(0, len(texts), chunk_size)) :
text_chunk = texts[i:i+chunk_size].tolist()
encs = tokenizer.encode_batch(text_chunk)
 all_ids.extend([enc.ids for enc in encs])
 return np.array(all_ids )<categorify>
train[train['title'].str.contains('Furious')]['title']
def regular_encode(texts, tokenizer, maxlen=512):
enc_di = tokenizer.batch_encode_plus(
 texts,
return_attention_masks=False,
return_token_type_ids=False,
pad_to_max_length=True,
max_length=maxlen 
)
 return np.array(enc_di['input_ids'] )<choose_model_class>
test[test['title'].fillna('Unknow' ).str.contains('Furious')]['title']
def build_model(transformer, max_len=512):

 input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(cls_token)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 return model<set_options>
d.remove(d[7] )
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<define_variables>
for i in d :
new_data['crew_contains_'+i[0][2]]=new_data['crew'].fillna('' ).apply(lambda x:1 if i[0][2] in x else 0)
AUTO = tf.data.experimental.AUTOTUNE
EPOCHS = 1
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
MAX_LEN = 275
MODEL = 'jplu/tf-xlm-roberta-large'<load_pretrained>
mon_is=[1,6,7,9,10,12]
for i in mon_is:
new_data['mon_is_'+str(i)]=new_data['release_month'].apply(lambda x:1 if x==i else 0 )
tokenizer = AutoTokenizer.from_pretrained(MODEL )<load_from_csv>
new_data['spoken_languages_count']=new_data['spoken_languages'].apply(lambda x:len(x))
train1 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv")
 train2 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv")
 train2.toxic = train2.toxic.round().astype(int)
 valid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')
 test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')
 submission = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv' )<concatenate>
new_data.loc[train.index].groupby('spoken_languages_count' ).revenue.median()
train = pd.concat([
train1[['comment_text', 'toxic']],
train2[['comment_text', 'toxic']].query('toxic==1'),
train2[['comment_text', 'toxic']].query('toxic==0' ).sample(n=100000, random_state=0)
 ] )<prepare_x_and_y>
status_du=pd.get_dummies(new_data['status'] )
%%time
x_train = regular_encode(train.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_valid = regular_encode(valid.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_test = regular_encode(test.content.values, tokenizer, maxlen=MAX_LEN)
 y_train = train.toxic.values
y_valid = valid.toxic.values<define_variables>
new_data=pd.concat([new_data,status_du],axis=1)
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(BATCH_SIZE)
 )<load_pretrained>
new_data=new_data.drop(['status'],axis=1)
 new_data.head()
%%time
with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model(transformer_layer, max_len=MAX_LEN)
 model.summary()<train_model>
new_data=new_data.drop(['original_language'],axis=1 )
EPOCHS = 1
n_steps = x_train.shape[0] // BATCH_SIZE
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS
)<train_model>
sum(new_data['geres_name'].isna() )
n_steps = x_valid.shape[0] // BATCH_SIZE
train_history_2 = model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
epochs=EPOCHS
)<predict_on_test>
new_data['geres_name']=new_data['geres_name'].fillna('Unknow' )
sub['toxic'] = model.predict(test_dataset, verbose=1)
<prepare_output>
new_data['geres_name_count']=new_data['geres_name'].apply(lambda x:len(x))
submission=sub[['id','toxic']]<save_to_csv>
ge_name_count_rev=new_data.loc[train.index].groupby(['geres_name_count'] ).revenue.median().sort_values()
submission.to_csv('submission.csv',index=False )<define_variables>
new_data=new_data.drop('homepage_end',axis=1 )
MAX_LEN = 192
LR = 1e-5
BATCH_SIZE = 16
TOTAL_STEPS_STAGE1 = 300
VALIDATE_EVERY_STAGE1 = 100
TOTAL_STEPS_STAGE2 = 200
VALIDATE_EVERY_STAGE2 = 100
PRETRAINED_MODEL = 'jplu/tf-xlm-roberta-large'
D = '/kaggle/input/jigsaw-multilingual-toxic-comment-classification/'
print(tf.__version__)
 logging.getLogger().setLevel(logging.NOTSET)
 AUTO = tf.data.experimental.AUTOTUNE<define_search_model>
new_data['original_and_new']=new_data['original_and_new'].apply(lambda x: 1 if x else 0 )
def connect_to_TPU() :

 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
global_batch_size = BATCH_SIZE * strategy.num_replicas_in_sync
return tpu, strategy, global_batch_size
tpu, strategy, global_batch_size = connect_to_TPU()
print("REPLICAS: ", strategy.num_replicas_in_sync )<load_from_csv>
list_geres_name=['Comedy','Thriller','Action','Drama','Romance']
train_df = pd.read_csv(D+'jigsaw-toxic-comment-train.csv')
 val_df = pd.read_csv(D+'validation.csv')
 test_df = pd.read_csv(D+'test.csv')
 sub_df = pd.read_csv(D+'sample_submission.csv')
 train_df = pd.concat([
train_df.query('toxic==1'),
train_df.query('toxic==0' ).sample(sum(train_df.toxic),random_state=42)
 ])
 train_df = train_df.sample(frac=1, random_state = 42 )<categorify>
for i in list_geres_name:
new_data['geres_name'+'_'+i]=new_data['geres_name'].apply(lambda x: 1 if i in x else 0 )
%%time
def regular_encode(texts, tokenizer, maxlen=512):
enc_di = tokenizer.batch_encode_plus(
 texts,
return_attention_masks=False,
return_token_type_ids=False,
pad_to_max_length=True,
max_length=maxlen 
)
 return np.array(enc_di['input_ids'])
 tokenizer = AutoTokenizer.from_pretrained(PRETRAINED_MODEL)
 X_train = regular_encode(train_df.comment_text.values, tokenizer, maxlen=MAX_LEN)
 X_val = regular_encode(val_df.comment_text.values, tokenizer, maxlen=MAX_LEN)
 X_test = regular_encode(test_df.content.values, tokenizer, maxlen=MAX_LEN)
 y_train = train_df.toxic.values.reshape(-1,1)
 y_val = val_df.toxic.values.reshape(-1,1 )<create_dataframe>
new_data=new_data.drop('geres_name',axis=1 )
def create_dist_dataset(X, y=None, training=False):
dataset = tf.data.Dataset.from_tensor_slices(X)
 if y is not None:
dataset_y = tf.data.Dataset.from_tensor_slices(y)
 dataset = tf.data.Dataset.zip(( dataset, dataset_y))
if training:
dataset = dataset.shuffle(len(X)).repeat()
dataset = dataset.batch(global_batch_size ).prefetch(AUTO)
 dist_dataset = strategy.experimental_distribute_dataset(dataset)
 return dist_dataset
train_dist_dataset = create_dist_dataset(X_train, y_train, True)
 val_dist_dataset = create_dist_dataset(X_val)
 test_dist_dataset = create_dist_dataset(X_test )<choose_model_class>
new_data['production_countries']=new_data['production_countries'].fillna('QQ' )
%%time
def create_model_and_optimizer() :
with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(PRETRAINED_MODEL)
 model = build_model(transformer_layer)
 optimizer = tf.keras.optimizers.Adam(learning_rate=LR, epsilon=1e-08)
 return model, optimizer
def build_model(transformer):
inp = Input(shape=(MAX_LEN,), dtype=tf.int32, name="input_word_ids")
 x = transformer(inp)[0][:, 0, :]
out = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=[inp], outputs=[out])
 return model
model, optimizer = create_model_and_optimizer()
model.summary()<compute_train_metric>
new_data['production_countries'].loc[new_data['production_countries']=='Unknow']['production_countries']='QQ'
def define_losses_and_metrics() :
with strategy.scope() :
loss_object = tf.keras.losses.BinaryCrossentropy(
 reduction=tf.keras.losses.Reduction.NONE, from_logits=False)
 def compute_loss(labels, predictions):
per_example_loss = loss_object(labels, predictions)
 loss = tf.nn.compute_average_loss(
 per_example_loss, global_batch_size = global_batch_size)
 return loss
train_accuracy_metric = tf.keras.metrics.AUC(name='training_AUC')
 return compute_loss, train_accuracy_metric
def train(train_dist_dataset, val_dist_dataset=None, y_val=None,
total_steps=5000, validate_every=500):
step = 0
for tensor in train_dist_dataset:
distributed_train_step(tensor)
 step+=1
if(step % validate_every == 0):
train_metric = train_accuracy_metric.result().numpy()
print("Step %d, train AUC: %.5f" %(step, train_metric))
if val_dist_dataset:
val_metric = roc_auc_score(y_val, predict(val_dist_dataset))
print(" validation AUC: %.5f" % val_metric)
 train_accuracy_metric.reset_states()
if step == total_steps:
break
@tf.function
def distributed_train_step(data):
strategy.experimental_run_v2(train_step, args=(data,))
def train_step(inputs):
features, labels = inputs
with tf.GradientTape() as tape:
predictions = model(features, training=True)
 loss = compute_loss(labels, predictions)
 gradients = tape.gradient(loss, model.trainable_variables)
 optimizer.apply_gradients(zip(gradients, model.trainable_variables))
train_accuracy_metric.update_state(labels, predictions)
 def predict(dataset):
predictions = []
for tensor in dataset:
predictions.append(distributed_prediction_step(tensor))
predictions = np.vstack(list(map(np.vstack,predictions)))
 return predictions
@tf.function
def distributed_prediction_step(data):
predictions = strategy.experimental_run_v2(prediction_step, args=(data,))
return strategy.experimental_local_results(predictions)
 def prediction_step(inputs):
features = inputs
predictions = model(features, training=False)
 return predictions
compute_loss, train_accuracy_metric = define_losses_and_metrics()<train_model>
list_pro_coun=list(new_data['production_countries'])
 d = Counter([j for i in list_pro_coun for j in i] ).most_common(5)
 d
%%time
train(train_dist_dataset, val_dist_dataset, y_val,
TOTAL_STEPS_STAGE1, VALIDATE_EVERY_STAGE1 )<train_model>
for i in d:
new_data['production_cou_name'+'_'+i[0]]=new_data['production_countries'].apply(lambda x: 1 if i[0] in x else 0 )
%%time
val_dist_dataset_4_training = create_dist_dataset(X_val, y_val, training=True)
 train(val_dist_dataset_4_training,
total_steps = TOTAL_STEPS_STAGE2,
validate_every = VALIDATE_EVERY_STAGE2 )<save_to_csv>
new_data['pro_country_count']=new_data['production_companies'].apply(lambda x:len(x))
%%time
sub_df['toxic'] = predict(test_dist_dataset)[:,0]
sub_df.to_csv('submission.csv', index=False )<import_modules>
pro_count_rev=new_data.loc[train.index].groupby('pro_country_count' ).revenue.median()
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
from kaggle_datasets import KaggleDatasets
import transformers
from transformers import TFAutoModel, AutoTokenizer
from tqdm.notebook import tqdm
from tokenizers import Tokenizer, models, pre_tokenizers, decoders, processors<categorify>
new_data=new_data.drop('production_countries',axis=1 )
def fast_encode(texts, tokenizer, chunk_size=256, maxlen=512):
tokenizer.enable_truncation(max_length=maxlen)
 tokenizer.enable_padding(max_length=maxlen)
 all_ids = []
for i in tqdm(range(0, len(texts), chunk_size)) :
text_chunk = texts[i:i+chunk_size].tolist()
encs = tokenizer.encode_batch(text_chunk)
 all_ids.extend([enc.ids for enc in encs])
 return np.array(all_ids )<categorify>
new_data['production_companies'].loc[new_data['production_companies']=='unknow']='u'
def regular_encode(texts, tokenizer, maxlen=512):
enc_di = tokenizer.batch_encode_plus(
 texts,
return_attention_masks=False,
return_token_type_ids=False,
pad_to_max_length=True,
max_length=maxlen 
)
 return np.array(enc_di['input_ids'] )<choose_model_class>
d.remove(( 'u',414))
def build_model(transformer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(cls_token)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 return model<set_options>
dd=[]
for i in d:
dd.append(i[0] )
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<define_variables>
dd=[]
for i in d:
dd.append(i[0] )
AUTO = tf.data.experimental.AUTOTUNE
EPOCHS = 2
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
MAX_LEN = 192
MODEL = 'jplu/tf-xlm-roberta-large'<load_pretrained>
new_data=new_data.drop('spoken_languages',axis=1 )
tokenizer = AutoTokenizer.from_pretrained(MODEL )<load_from_csv>
new_data['runtime'].loc[new_data['runtime']==0]=1
new_data['time_budget']=new_data['budget'].fillna(0)/new_data['runtime'].fillna(1 )
train1 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv")
 train2 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv")
 train2.toxic = train2.toxic.round().astype(int)
 valid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')
 test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv' )<concatenate>
new_data=new_data.drop('production_companies',axis=1 )
train = pd.concat([
train1[['comment_text', 'toxic']],
train2[['comment_text', 'toxic']].query('toxic==1'),
train2[['comment_text', 'toxic']].query('toxic==0' ).sample(n=100000, random_state=0)
 ] )<prepare_x_and_y>
new_data=new_data.drop('crew',axis=1 )
%%time
x_train = regular_encode(train.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_valid = regular_encode(valid.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_test = regular_encode(test.content.values, tokenizer, maxlen=MAX_LEN)
 y_train = train.toxic.values
y_valid = valid.toxic.values<categorify>
for i in list1:
new_data['title_is'+i]=new_data['title'].fillna('' ).apply(lambda x: 1 if i in x else 0 )
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(BATCH_SIZE)
 )<train_model>
for col in ['tagline', 'overview','title']:
new_data['len_' + col] =new_data[col].fillna('' ).apply(lambda x: len(x))
new_data['words_' + col] = new_data[col].fillna('' ).apply(lambda x: len(x.split(' ')))
 new_data=new_data.drop(col,axis=1 )
n_steps = x_train.shape[0] // BATCH_SIZE
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS
)<train_model>
new_data['budget_popularity']=new_data['budget']*1.0/new_data['popularity']
n_steps = x_valid.shape[0] // BATCH_SIZE
train_history_2 = model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
epochs=EPOCHS
)<save_to_csv>
a=new_data.groupby(['release_year'] ).release_month.value_counts()
b=new_data.groupby(['release_year','release_month'] ).release_day.value_counts()
new_data[(new_data['release_year']==2017)&(new_data['release_day']==30)]
sub['toxic'] = model.predict(test_dataset, verbose=1)
 sub.to_csv('submission.csv', index=False )<import_modules>
new_data['_releaseYear_popularity_ratio'] = new_data['release_year'] / new_data['popularity']
new_data['runtime_to_mean_year'] = new_data['runtime'] / new_data.groupby("release_year")["runtime"].transform('mean')
 new_data['popularity_to_mean_year'] = new_data['popularity'] / new_data.groupby("release_year")["popularity"].transform('mean')
 new_data['budget_to_mean_year'] = new_data['budget'] / new_data.groupby("release_year")["budget"].transform('mean')
import numpy as np
import pandas as pd
import xgboost as xgb
import matplotlib.pyplot as plt<define_variables>
macro_cols = ["balance_trade", "balance_trade_growth", "eurrub", "average_provision_of_build_contract",
"micex_rgbi_tr", "micex_cbi_tr", "deposits_rate", "mortgage_value", "mortgage_rate",
"income_per_cap", "rent_price_4+room_bus", "museum_visitis_per_100_cap", "apartment_build"]<load_from_csv>
df_train = pd.read_csv(".. /input/train.csv", parse_dates=['timestamp'])
 df_test = pd.read_csv(".. /input/test.csv", parse_dates=['timestamp'])
 df_macro = pd.read_csv(".. /input/macro.csv", parse_dates=['timestamp'], usecols=['timestamp'] + macro_cols)
 df_train.head()<merge>
new_train=new_data.loc[train.index]
new_test=new_data.loc[test.index]
ylog_train_all = np.log1p(df_train['price_doc'].values)
 id_test = df_test['id']
df_train.drop(['id', 'price_doc'], axis=1, inplace=True)
 df_test.drop(['id'], axis=1, inplace=True)
 num_train = len(df_train)
 df_all = pd.concat([df_train, df_test])
 df_all = pd.merge_ordered(df_all, df_macro, on='timestamp', how='left')
 print(df_all.shape)
 month_year =(df_all.timestamp.dt.month + df_all.timestamp.dt.year * 100)
 month_year_cnt_map = month_year.value_counts().to_dict()
df_all['month_year_cnt'] = month_year.map(month_year_cnt_map)
 week_year =(df_all.timestamp.dt.weekofyear + df_all.timestamp.dt.year * 100)
 week_year_cnt_map = week_year.value_counts().to_dict()
df_all['week_year_cnt'] = week_year.map(week_year_cnt_map)
 df_all['month'] = df_all.timestamp.dt.month
df_all['dow'] = df_all.timestamp.dt.dayofweek
df_all['rel_floor'] = df_all['floor'] / df_all['max_floor'].astype(float)
 df_all['rel_kitch_sq'] = df_all['kitch_sq'] / df_all['full_sq'].astype(float)
 df_all.drop(['timestamp'], axis=1, inplace=True )<concatenate>
x=new_train.drop('revenue',axis=1)
 y=new_train['revenue']
x=x.fillna(0)
 decision_tree = DecisionTreeRegressor(max_depth = 6)
 decision_tree.fit(x, y)
df_numeric = df_all.select_dtypes(exclude=['object'])
 df_obj = df_all.select_dtypes(include=['object'] ).copy()
for c in df_obj:
df_obj[c] = pd.factorize(df_obj[c])[0]
df_values = pd.concat([df_numeric, df_obj], axis=1 )<prepare_x_and_y>
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2, random_state=42)
 model_XG = xgboost.XGBRegressor()
model_XG.fit(x_train,y_train)
 y_predict_rf = model_XG.predict(x_test)
 print(mean_squared_error(y_test, y_predict_rf))
dtrain_all = xgb.DMatrix(X_train_all, ylog_train_all, feature_names=df_columns)
 dtrain = xgb.DMatrix(X_train, ylog_train, feature_names=df_columns)
 dval = xgb.DMatrix(X_val, ylog_val, feature_names=df_columns)
 dtest = xgb.DMatrix(X_test, feature_names=df_columns )<train_model>
X_test=new_test.drop('revenue',axis=1)
 X_test=X_test.fillna(0 )
xgb_params = {
'eta': 0.05,
'max_depth': 5,
'subsample': 1.0,
'colsample_bytree': 0.7,
'objective': 'reg:linear',
'eval_metric': 'rmse',
'silent': 1
}
partial_model = xgb.train(xgb_params, dtrain, num_boost_round=1000, evals=[(dval, 'val')],
early_stopping_rounds=20, verbose_eval=20)
 num_boost_round = partial_model.best_iteration<find_best_params>
ypred=model_XG.predict(X_test )
num_boost_round = partial_model.best_iteration<train_model>
dtrain = xgb.DMatrix(x, y)
 dtest = xgb.DMatrix(X_test )
model = xgb.train(dict(xgb_params, silent=0), dtrain_all, num_boost_round=num_boost_round )<save_to_csv>
xgb_params = {
'eta': 0.08,
'max_depth': 15,
'subsample': 0.7,
'colsample_bytree': 0.7,
'objective': 'reg:linear',
'eval_metric': 'rmse',
'silent': 1
}
cv_output = xgb.cv(xgb_params, dtrain, num_boost_round=1000, early_stopping_rounds=400, verbose_eval=50, show_stdv=False)
 cv_output[['train-rmse-mean', 'test-rmse-mean']].plot()
ylog_pred = model.predict(dtest)
 y_pred = np.exp(ylog_pred)- 1
df_sub = pd.DataFrame({'id': id_test, 'price_doc': y_pred})
 df_sub.to_csv('sub.csv', index=False )<set_options>
y_predict1 = abs(model.predict(dtest))
sns.set()<load_from_csv>
train=x
y_train=np.log1p(y)
 test=X_test
y=y_train
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv')
<count_missing_values>
n_folds = 5
def rmsle_cv(model):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(train.values)
 rmse = np.sqrt(-cross_val_score(model, train.values,y_train.values, scoring="neg_mean_squared_error", cv=kf))
return(rmse)
 def eval_model(model, name):
start_time = time.time()
score = rmsle_cv(model)
 print("{} score: {:.4f}({:.4f}), execution time: {:.1f}".format(name, score.mean() , score.std() , time.time() -start_time))
train_df.isnull().sum() [train_df.isnull().sum() > 0]<import_modules>
mod_lasso = make_pipeline(RobustScaler() , Lasso(alpha=0.005, random_state=1))
eval_model(mod_lasso, "lasso")
 mod_enet = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3))
eval_model(mod_enet, "enet")
 mod_cat = CatBoostRegressor(iterations=10000, learning_rate=0.01,
depth=5, eval_metric='RMSE',
colsample_bylevel=0.7, random_seed = 17, silent=True,
bagging_temperature = 0.2, early_stopping_rounds=200)
 eval_model(mod_cat, "cat")
 mod_gboost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.03,
max_depth=10, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state=5)
 eval_model(mod_gboost, "gboost")
 mod_xgb = xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.03, max_depth=10,
min_child_weight=1.7817, n_estimators=2200,
reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213, silent=1,
random_state=7, nthread=-1)
 eval_model(mod_xgb, "xgb")
 mod_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=1023,
learning_rate=0.03, n_estimators=650,
max_bin=58, bagging_fraction=0.80,max_depth=10,
bagging_freq=5, feature_fraction=0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf=7, min_sum_hessian_in_leaf=11)
 eval_model(mod_lgb, "lgb" )
import missingno as msno<feature_engineering>
class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models, meta_model, n_folds=5):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, X, y):
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=156)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X[train_index], y[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 return self.meta_model_.predict(meta_features)
 mod_stacked = StackingAveragedModels(base_models =(mod_cat, mod_xgb, mod_gboost, mod_lgb), meta_model = mod_enet)
 eval_model(mod_stacked, "stacked" )
train_df.loc[(pd.isnull(train_df['v2a1'])& train_df['tipovivi1'] == 1), 'v2a1'] = 0<filter>
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
def predict(model):
model.fit(train.values,y_train.values)
 train_pred = model.predict(train.values)
 pred = np.expm1(model.predict(test.values))
print(rmsle(y_train, train_pred))
return(pred )
train_df = train_df.loc[pd.notnull(train_df['v2a1'])]<count_values>
pre_lasso = predict(mod_lasso)
 pre_enet = predict(mod_enet)
 pre_xgb = predict(mod_xgb)
 pre_gboost = predict(mod_gboost)
 pre_lgb = predict(mod_lgb)
 pre_stack1 = predict(mod_stacked )
<feature_engineering><EOS> 
new_test['id']=new_test.index
new_test['revenue']=pre_stack1
new_test[['id','revenue']].to_csv('submission_Dragon2.csv', index=False)
 new_test[['id','revenue']].head()