Kaggle competition: TMDB Box Office Prediction
valid_files=list(filter(lambda file:"test" not in file and "nen1" in file and "Sentences" not in file, data_files))
valid_files<define_variables>
train_data = pd.read_csv('.. /input/train.csv', sep=None, engine='python')
 test_data = pd.read_csv('.. /input/test.csv', sep=None, engine='python' )
if IN_COLAB:
TRANLSATED_DATA_PATH_ONE=root+"/translated-train-bias-all-langs"
TRANLSATED_DATA_PATH_TWO=root+"/jigsaw-train-multilingual-coments-google-api"
elif not localEnvironment:
TRANSLATED_DATA_PATH = KaggleDatasets().get_gcs_path("jigsawtranslated")+"/tfrecords-translated"
else:
pass
TRANSLATED_DATA_PATH<define_variables>
m_train, n1 = train_data.shape
m_test, n2 = test_data.shape
print("{} training samples".format(m_train))
print("{} test samples".format(m_test))
print("{} features to t".format(n1))
toxic_translated_files=list(filter(lambda file:"nen" not in file and "toxic" in file and "non" not in file and "valid" not in file and "sharded" not in file, translated_files))
random.choice(toxic_translated_files )<define_variables>
print("- The target is:
{} ".format([item for item in train_data.columns.tolist() if item not in test_data.columns.tolist() ]))
print(train_data["".join([item for item in train_data.columns.tolist() if item not in test_data.columns.tolist() ])].dtypes )
non_toxic_translated_files=list(filter(lambda file:"nen" not in file and "toxic" in file and "non" in file and "valid" not in file and "sharded" not in file, translated_files))
random.choice(non_toxic_translated_files )<define_variables>
train_data.drop(columns=['homepage', 'imdb_id', 'original_title', 'poster_path'], inplace=True)
 test_data.drop(columns=['homepage', 'imdb_id', 'original_title', 'poster_path'], inplace=True)
 train_data.budget = train_data.budget.astype(float)
 train_data.revenue = train_data.revenue.astype(float)
 test_data.budget = test_data.budget.astype(float )
<define_variables>
color = {'boxes': 'DarkGreen', 'whiskers': 'DarkOrange','medians': 'DarkBlue', 'caps': 'Gray'}
plt.figure(figsize=(20,10))
belongs_df = train_data[['belongs_to_collection', 'revenue']]
belongs_df['belongs_to_collection_bool'] = True
belongs_df.belongs_to_collection_bool[train_data.belongs_to_collection.isnull() ] = False
belongs_df['in_saga'] = np.nan
belongs_df.in_saga[belongs_df.belongs_to_collection_bool == True] = belongs_df.revenue[belongs_df.belongs_to_collection_bool == True]
belongs_df['no_saga'] = np.nan
belongs_df.no_saga[belongs_df.belongs_to_collection_bool == False] = belongs_df.revenue[belongs_df.belongs_to_collection_bool == False]
belongs_df.belongs_to_collection[~train_data.belongs_to_collection.isnull() ] = train_data.belongs_to_collection[~train_data.belongs_to_collection.isnull() ].str.split("'name': '" ).str[1].str.split("', 'poster" ).str[0]
belongs_df.drop(columns=['revenue', 'belongs_to_collection', 'belongs_to_collection_bool'], inplace=True)
 print("Movies in saga: {}/{}={}% ".format(len(~belongs_df.in_saga.isnull()), len(belongs_df), 100.0*len(~belongs_df.in_saga.isnull())/len(belongs_df)))
 sns.boxplot(data=belongs_df, orient='h')
 plt.xlabel('value')
 plt.ylabel('Data')
 plt.title('Box plot of Belongig to Collection')
 plt.show()
opus_langs=[['en', 'it'],
['en', 'ru'],
['en', 'tr'],
['en', 'pt_br'],
['en', 'fr'],
['en', 'pt'],
['en', 'es'],
['en', 'hi']]
if IN_COLAB:
OPUS_DS_PATH=root+"opus"
elif not localEnvironment:
OPUS_DS_PATH = KaggleDatasets().get_gcs_path("opus-for-jigsaw-sharded")
 else:
OPUS_DS_PATH = ".. /opus-processed"
OPUS_DS_PATH<define_variables>
genres_df = train_data[['genres', 'revenue']]
genres_df['genres_test'] = genres_df['genres']
genres_df.genres_test = genres_df.genres_test.str.strip('[]')
 genres_df.genres_test[genres_df.genres_test.isnull() ] = ''
genres_list = pd.Series(list(set(", ".join(genres_df.genres_test.unique().tolist() ).split('}, ')))).str.split("'name': '" ).str[1].str.split("'" ).str[0].tolist()
for i, genre in enumerate(genres_list):
genres_df[genre] = np.nan
genres_df[genre][genres_df.genres_test.str.contains(genre)] = genres_df.revenue[genres_df.genres_test.str.contains(genre)]
genres_df.drop(columns=['revenue', 'genres', 'genres_test'], inplace=True)
 plt.figure(figsize=(20,12))
sns.boxplot(data=genres_df)
 plt.xlabel('Data')
 plt.ylabel('Value')
 plt.title('Box plot of genres')
 plt.show()
<data_type_conversions><EOS> 
title_df = train_data[['title', 'belongs_to_collection']].sort_values('title')
 print("Non-null titles: {}, Non-null unique titles: {} ".format(len(title_df.title), len(title_df.title.unique())))
 print("Duplicated movies: {} ".format(title_df.title[title_df.duplicated(keep='first')].tolist()))
print("Titles that belong to a collection {} ".format(len(title_df[~title_df.belongs_to_collection.isnull() ])))
 number_tuple =(' 0', ' 1', ' 2', ' 3', ' 4', ' 5', ' 6', ' 7', ' 8', ' 9')
 title_df = title_df[(~title_df.belongs_to_collection.isnull())&(title_df.title.str.endswith(number_tuple)) ]
print("Titles that belong to a collection and indicate the number of movie {} ".format(len(title_df)) )