Kaggle competition: IEEE-CIS Fraud Detection
title_df = train_data[['title', 'belongs_to_collection']].sort_values('title')
 print("Non-null titles: {}, Non-null unique titles: {} ".format(len(title_df.title), len(title_df.title.unique())))
 print("Duplicated movies: {} ".format(title_df.title[title_df.duplicated(keep='first')].tolist()))
print("Titles that belong to a collection {} ".format(len(title_df[~title_df.belongs_to_collection.isnull() ])))
 number_tuple =(' 0', ' 1', ' 2', ' 3', ' 4', ' 5', ' 6', ' 7', ' 8', ' 9')
 title_df = title_df[(~title_df.belongs_to_collection.isnull())&(title_df.title.str.endswith(number_tuple)) ]
print("Titles that belong to a collection and indicate the number of movie {} ".format(len(title_df)) )<set_options>
!pip install missingpy
pd.set_option('max_columns', None)
 %matplotlib inline
plt.style.use('ggplot')
 stop = set(stopwords.words('english'))
warnings.filterwarnings("ignore" )<load_from_csv>
!pip install git+https://github.com/t0biK/data_science_utils.git
train=pd.read_csv('.. /input/train.csv',index_col=0)
 test=pd.read_csv('.. /input/test.csv',index_col=0)
 train.head()<feature_engineering>
%matplotlib inline
!pip install --upgrade pip
!pip install imdbpy
ia=IMDb()
start =time.clock()
def fill_col(df,col,key,fill_missing_type,nested_arr):
indexes=df.index[df[col]!=df[col]].tolist()
for index in indexes:
imdb_id=df.loc[index]['imdb_id']
movie=ia.get_movie(imdb_id[2:])
 if key in movie:
if(nested_arr):
arr=[]
for el in movie[key]:
arr.append(str(el))
df.at[index, col] = ','.join(arr)
 else:
df.at[index, col] = ','.join(movie[key])
 else:
if(fill_missing_type=='mean'):
df.at[index, col] = df[col].mean()
elif(fill_missing_type=='others'):
df.at[index, col] = 'others'
elif(fill_missing_type=='None'):
df.at[index, col] = None
list1=['runtime','original_language','popularity','title','crew']
list2=['genres','spoken_languages']
for i in list1:
print(i)
 fill_col(train,i,i+'s','None',False)
 fill_col(test,i,i+'s','None',False)
 for j in list2:
print(j)
 fill_col(train,j,j,'None',False)
 fill_col(test,j,j,'None',False)
 end = time.clock()
print('Running time: %s Seconds'%(end-start)) 
<feature_engineering>
test_identity = pd.read_csv('/kaggle/input/ieee-fraud-detection/test_identity.csv')
 test_identity.columns = test_identity.columns.str.replace("-", "_")
 train_identity = pd.read_csv('/kaggle/input/ieee-fraud-detection/train_identity.csv')
 test_transaction = pd.read_csv('/kaggle/input/ieee-fraud-detection/test_transaction.csv')
 train_transaction = pd.read_csv('/kaggle/input/ieee-fraud-detection/train_transaction.csv' )
test['revenue']=-99<feature_engineering>
train = pd.merge(train_identity, train_transaction, on='TransactionID', how='right')
 test = pd.merge(test_identity, test_transaction, on='TransactionID', how='right' )
train['be_is_na']=train['belongs_to_collection'].apply(lambda x :1 if x is np.nan else 0)
 test['be_is_na']=test['belongs_to_collection'].apply(lambda x :1 if x is np.nan else 0)
 train=train.drop('belongs_to_collection',axis=1)
 test=test.drop('belongs_to_collection',axis=1 )<feature_engineering>
test_identity, train_identity, test_transaction, train_transaction = [None]*4
train.loc[train.index == 16,'revenue'] = 192864
train.loc[train.index == 90,'budget'] = 30000000
train.loc[train.index== 118,'budget'] = 60000000
train.loc[train.index== 149,'budget'] = 18000000
train.loc[train.index== 313,'revenue'] = 12000000
train.loc[train.index == 451,'revenue'] = 12000000
train.loc[train.index == 464,'budget'] = 20000000
train.loc[train.index == 470,'budget'] = 13000000
train.loc[train.index== 513,'budget'] = 930000
train.loc[train.index == 797,'budget'] = 8000000
train.loc[train.index == 819,'budget'] = 90000000
train.loc[train.index == 850,'budget'] = 90000000
train.loc[train.index == 1007,'budget'] = 2
train.loc[train.index== 1112,'budget'] = 7500000
train.loc[train.index == 1131,'budget'] = 4300000
train.loc[train.index == 1359,'budget'] = 10000000
train.loc[train.index == 1542,'budget'] = 1
train.loc[train.index == 1570,'budget'] = 15800000
train.loc[train.index== 1571,'budget'] = 4000000
train.loc[train.index == 1714,'budget'] = 46000000
train.loc[train.index == 1721,'budget'] = 17500000
train.loc[train.index== 1865,'revenue'] = 25000000
train.loc[train.index == 1885,'budget'] = 12
train.loc[train.index == 2091,'budget'] = 10
train.loc[train.index == 2268,'budget'] = 17500000
train.loc[train.index == 2491,'budget'] = 6
train.loc[train.index == 2602,'budget'] = 31000000
train.loc[train.index == 2612,'budget'] = 15000000
train.loc[train.index == 2696,'budget'] = 10000000
train.loc[train.index == 2801,'budget'] = 10000000
train.loc[train.index == 335,'budget'] = 2
train.loc[train.index == 348,'budget'] = 12
train.loc[train.index == 470,'budget'] = 13000000
train.loc[train.index == 513,'budget'] = 1100000
train.loc[train.index == 640,'budget'] = 6
train.loc[train.index == 696,'budget'] = 1
train.loc[train.index == 797,'budget'] = 8000000
train.loc[train.index == 850,'budget'] = 1500000
train.loc[train.index == 1199,'budget'] = 5
train.loc[train.index == 1282,'budget'] = 9
train.loc[train.index== 1347,'budget'] = 1
train.loc[train.index== 1755,'budget'] = 2
train.loc[train.index == 1801,'budget'] = 5
train.loc[train.index == 1918,'budget'] = 592
train.loc[train.index == 2033,'budget'] = 4
train.loc[train.index == 2118,'budget'] = 344
train.loc[train.index == 2252,'budget'] = 130
train.loc[train.index == 2256,'budget'] = 1
train.loc[train.index == 2696,'budget'] = 10000000<feature_engineering>
numerical_columns = ['id_0'+str(i)for i in range(1,10)]+['id_10', 'id_11']
test.loc[test.index== 3033,'budget'] = 250
test.loc[test.index== 3051,'budget'] = 50
test.loc[test.index == 3084,'budget'] = 337
test.loc[test.index == 3224,'budget'] = 4
test.loc[test.index == 3594,'budget'] = 25
test.loc[test.index == 3619,'budget'] = 500
test.loc[test.index == 3831,'budget'] = 3
test.loc[test.index== 3935,'budget'] = 500
test.loc[test.index == 4049,'budget'] = 995946
test.loc[test.index== 4424,'budget'] = 3
test.loc[test.index == 4460,'budget'] = 8
test.loc[test.index == 4555,'budget'] = 1200000
test.loc[test.index== 4624,'budget'] = 30
test.loc[test.index== 4645,'budget'] = 500
test.loc[test.index == 4709,'budget'] = 450
test.loc[test.index == 4839,'budget'] = 7
test.loc[test.index== 3125,'budget'] = 25
test.loc[test.index== 3142,'budget'] = 1
test.loc[test.index == 3201,'budget'] = 450
test.loc[test.index == 3222,'budget'] = 6
test.loc[test.index== 3545,'budget'] = 38
test.loc[test.index == 3670,'budget'] = 18
test.loc[test.index == 3792,'budget'] = 19
test.loc[test.index == 3881,'budget'] = 7
test.loc[test.index == 3969,'budget'] = 400
test.loc[test.index == 4196,'budget'] = 6
test.loc[test.index == 4221,'budget'] = 11
test.loc[test.index == 4222,'budget'] = 500
test.loc[test.index== 4285,'budget'] = 11
test.loc[test.index == 4319,'budget'] = 1
test.loc[test.index == 4639,'budget'] = 10
test.loc[test.index == 4719,'budget'] = 45
test.loc[test.index == 4822,'budget'] = 22
test.loc[test.index == 4829,'budget'] = 20
test.loc[test.index== 4969,'budget'] = 20
test.loc[test.index== 5021,'budget'] = 40
test.loc[test.index== 5035,'budget'] = 1
test.loc[test.index== 5063,'budget'] = 14
test.loc[test.index == 5119,'budget'] = 2
test.loc[test.index== 5214,'budget'] = 30
test.loc[test.index== 5221,'budget'] = 50
test.loc[test.index== 4903,'budget'] = 15
test.loc[test.index == 4983,'budget'] = 3
test.loc[test.index == 5102,'budget'] = 28
test.loc[test.index== 5217,'budget'] = 75
test.loc[test.index == 5224,'budget'] = 3
test.loc[test.index== 5469,'budget'] = 20
test.loc[test.index == 5840,'budget'] = 1
test.loc[test.index == 5960,'budget'] = 30
test.loc[test.index == 6506,'budget'] = 11
test.loc[test.index== 6553,'budget'] = 280
test.loc[test.index == 6561,'budget'] = 7
test.loc[test.index== 6582,'budget'] = 218
test.loc[test.index == 6638,'budget'] = 5
test.loc[test.index== 6749,'budget'] = 8
test.loc[test.index==6759,'budget'] = 50
test.loc[test.index == 6856,'budget'] = 10
test.loc[test.index== 6858,'budget'] = 100
test.loc[test.index == 6876,'budget'] = 250
test.loc[test.index == 6972,'budget'] = 1
test.loc[test.index== 7079,'budget'] = 8000000
test.loc[test.index == 7150,'budget'] = 118
test.loc[test.index == 6506,'budget'] = 118
test.loc[test.index == 7225,'budget'] = 6
test.loc[test.index == 7231,'budget'] = 85
test.loc[test.index == 5222,'budget'] = 5
test.loc[test.index == 5322,'budget'] = 90
test.loc[test.index == 5350,'budget'] = 70
test.loc[test.index == 5378,'budget'] = 10
test.loc[test.index== 5545,'budget'] = 80
test.loc[test.index == 5810,'budget'] = 8
test.loc[test.index== 5926,'budget'] = 300
test.loc[test.index== 5927,'budget'] = 4
test.loc[test.index== 5986,'budget'] = 1
test.loc[test.index == 6053,'budget'] = 20
test.loc[test.index== 6104,'budget'] = 1
test.loc[test.index == 6130,'budget'] = 30
test.loc[test.index == 6301,'budget'] = 150
test.loc[test.index == 6276,'budget'] = 100
test.loc[test.index == 6473,'budget'] = 100
test.loc[test.index== 6842,'budget'] = 30<concatenate>
test.loc[:,~test.columns.isin(numerical_columns)] = test.loc[:,~test.columns.isin(numerical_columns)].fillna(-999)
 train.loc[:,~train.columns.isin(numerical_columns)] = train.loc[:,~train.columns.isin(numerical_columns)].fillna(-999 )
new_data=pd.concat([train,test],axis=0)
 new_data.head()<count_missing_values>
imputer = MissForest()
test.loc[:,test.columns.isin(numerical_columns)] = imputer.fit_transform(test.loc[:,test.columns.isin(numerical_columns)])
 train.loc[:,train.columns.isin(numerical_columns)] = imputer.fit_transform(train.loc[:,train.columns.isin(numerical_columns)] )
new_data.isnull().sum()<filter>
params = {'objective': 'binary', "boosting_type": "gbdt", "subsample": 1, "bagging_seed": 11, "metric": 'auc', 'num_boost_round':100, 'verbose':-1}
new_data[new_data['release_date'].isnull() ]<feature_engineering>
useful_features = ['TransactionAmt', 'ProductCD', 'card1', 'card2', 'card3', 'card4', 'card5', 'card6', 'addr1', 'addr2', 'dist1',
'P_emaildomain', 'R_emaildomain', 'C1', 'C2', 'C4', 'C5', 'C6', 'C7', 'C8', 'C9', 'C10', 'C11', 'C12', 'C13',
'C14', 'D1', 'D2', 'D3', 'D4', 'D5', 'D6', 'D8', 'D9', 'D10', 'D11', 'D12', 'D13', 'D14', 'D15', 'M2', 'M3',
'M4', 'M5', 'M6', 'M7', 'M8', 'M9', 'V3', 'V4', 'V5', 'V6', 'V7', 'V8', 'V9', 'V10', 'V11', 'V12', 'V13', 'V17',
'V19', 'V20', 'V29', 'V30', 'V33', 'V34', 'V35', 'V36', 'V37', 'V38', 'V40', 'V44', 'V45', 'V46', 'V47', 'V48',
'V49', 'V51', 'V52', 'V53', 'V54', 'V56', 'V58', 'V59', 'V60', 'V61', 'V62', 'V63', 'V64', 'V69', 'V70', 'V71',
'V72', 'V73', 'V74', 'V75', 'V76', 'V78', 'V80', 'V81', 'V82', 'V83', 'V84', 'V85', 'V87', 'V90', 'V91', 'V92',
'V93', 'V94', 'V95', 'V96', 'V97', 'V99', 'V100', 'V126', 'V127', 'V128', 'V130', 'V131', 'V138', 'V139', 'V140',
'V143', 'V145', 'V146', 'V147', 'V149', 'V150', 'V151', 'V152', 'V154', 'V156', 'V158', 'V159', 'V160', 'V161',
'V162', 'V163', 'V164', 'V165', 'V166', 'V167', 'V169', 'V170', 'V171', 'V172', 'V173', 'V175', 'V176', 'V177',
'V178', 'V180', 'V182', 'V184', 'V187', 'V188', 'V189', 'V195', 'V197', 'V200', 'V201', 'V202', 'V203', 'V204',
'V205', 'V206', 'V207', 'V208', 'V209', 'V210', 'V212', 'V213', 'V214', 'V215', 'V216', 'V217', 'V219', 'V220',
'V221', 'V222', 'V223', 'V224', 'V225', 'V226', 'V227', 'V228', 'V229', 'V231', 'V233', 'V234', 'V238', 'V239',
'V242', 'V243', 'V244', 'V245', 'V246', 'V247', 'V249', 'V251', 'V253', 'V256', 'V257', 'V258', 'V259', 'V261',
'V262', 'V263', 'V264', 'V265', 'V266', 'V267', 'V268', 'V270', 'V271', 'V272', 'V273', 'V274', 'V275', 'V276',
'V277', 'V278', 'V279', 'V280', 'V282', 'V283', 'V285', 'V287', 'V288', 'V289', 'V291', 'V292', 'V294', 'V303',
'V304', 'V306', 'V307', 'V308', 'V310', 'V312', 'V313', 'V314', 'V315', 'V317', 'V322', 'V323', 'V324', 'V326',
'V329', 'V331', 'V332', 'V333', 'V335', 'V336', 'V338', 'id_01', 'id_02', 'id_03', 'id_05', 'id_06', 'id_09',
'id_11', 'id_12', 'id_13', 'id_14', 'id_15', 'id_17', 'id_19', 'id_20', 'id_30', 'id_31', 'id_32', 'id_33',
'id_36', 'id_37', 'id_38', 'DeviceType', 'DeviceInfo']
new_data['release_date']=new_data['release_date'].fillna('3/20/01' )<feature_engineering>
train['TransactionAmt_decimal'] =(( train['TransactionAmt'] - train['TransactionAmt'].astype(int)) * 1000 ).astype(int)
 test['TransactionAmt_decimal'] =(( test['TransactionAmt'] - test['TransactionAmt'].astype(int)) * 1000 ).astype(int)
 train['card1_count_full'] = train['card1'].map(pd.concat([train['card1'], test['card1']], ignore_index=True ).value_counts(dropna=False))
test['card1_count_full'] = test['card1'].map(pd.concat([train['card1'], test['card1']], ignore_index=True ).value_counts(dropna=False))
train['Transaction_day_of_week'] = np.floor(( train['TransactionDT'] /(3600 * 24)- 1)% 7)
 test['Transaction_day_of_week'] = np.floor(( test['TransactionDT'] /(3600 * 24)- 1)% 7)
 train['Transaction_hour'] = np.floor(train['TransactionDT'] / 3600)% 24
test['Transaction_hour'] = np.floor(test['TransactionDT'] / 3600)% 24
for feature in ['id_02__id_20', 'id_02__D8', 'D11__DeviceInfo', 'DeviceInfo__P_emaildomain', 'P_emaildomain__C2',
'card2__dist1', 'card1__card5', 'card2__id_20', 'card5__P_emaildomain', 'addr1__card1']:
f1, f2 = feature.split('__')
 train[feature] = train[f1].astype(str)+ '_' + train[f2].astype(str)
 test[feature] = test[f1].astype(str)+ '_' + test[f2].astype(str)
 le = LabelEncoder()
le.fit(list(train[feature].astype(str ).values)+ list(test[feature].astype(str ).values))
train[feature] = le.transform(list(train[feature].astype(str ).values))
test[feature] = le.transform(list(test[feature].astype(str ).values))
for feature in ['id_34', 'id_36']:
if feature in useful_features:
train[feature + '_count_full'] = train[feature].map(pd.concat([train[feature], test[feature]], ignore_index=True ).value_counts(dropna=False))
test[feature + '_count_full'] = test[feature].map(pd.concat([train[feature], test[feature]], ignore_index=True ).value_counts(dropna=False))
for feature in ['id_01', 'id_31', 'id_33', 'id_35', 'id_36']:
if feature in useful_features:
train[feature + '_count_dist'] = train[feature].map(train[feature].value_counts(dropna=False))
test[feature + '_count_dist'] = test[feature].map(test[feature].value_counts(dropna=False))
new_data['release_year']=pd.to_datetime(new_data['release_date'] ).dt.year
new_data['release_month']=pd.to_datetime(new_data['release_date'] ).dt.month
new_data['release_day']=pd.to_datetime(new_data['release_date'] ).dt.day
new_data['release_dayofweek'] = pd.to_datetime(new_data['release_date'] ).dt.dayofweek
new_data['release_quarter'] = pd.to_datetime(new_data['release_date'] ).dt.quarter<feature_engineering>
params = {'num_leaves': 491,
'min_child_weight': 0.03454472573214212,
'feature_fraction': 0.3797454081646243,
'bagging_fraction': 0.4181193142567742,
'min_data_in_leaf': 106,
'objective': 'binary',
'max_depth': -1,
'learning_rate': 0.0066,
'n_estimators': 1000,
"boosting_type": "gbdt",
"bagging_seed": 11,
"metric": 'auc',
"verbosity": -1,
'reg_alpha': 0.3899927210061127,
'reg_lambda': 0.6485237330340494,
'random_state': 47
}
new_data['release_year'].loc[new_data['release_year']>=2018]-=100<drop_column>
X_train, X_test, y_train, y_test = train_test_split(train.drop(columns=['isFraud', 'TransactionID', 'TransactionDT']), train['isFraud'], test_size=0.33 )
new_data=new_data.drop('release_date',axis=1 )<define_variables>
for col in X_train.select_dtypes(include='object' ).columns:
le = LabelEncoder()
le.fit(list(X_train[col].astype(str ).values)+ list(X_test[col].astype(str ).values)+ list(test[col].astype(str ).values))
X_train[col] = le.transform(list(X_train[col].astype(str ).values))
X_test[col] = le.transform(list(X_test[col].astype(str ).values))
test[col] = le.transform(list(test[col].astype(str ).values))
weiji=[1929,1930,1930,1931,1932,1933,1934,1935,1936,1937,1938,1939,1940,1941,1942,1943,1944,1945,1950,1951,1952,1953,1961,1962,1963,1964,1965,1966,1967,1968,1969,1971,1973,1974,1975,1980,1981,1982,1983,1984,1985,1986,1987,1988,1989,
1990,1991,1992,1993,1994,1995,1996,1997,1998,1999,2007,2008,2009,2010,2011]<feature_engineering>
clf = lgb.LGBMClassifier(**params)
 clf.fit(X_train, y_train )
for i in weiji:
new_data['is_'+str(i)]=new_data['release_year'].apply(lambda x:1 if x==i else 0 )<feature_engineering>
cols_to_drop = ['TransactionID', 'TransactionDT']
new_data['homepage_fact']=new_data['homepage'].apply(lambda x: 0 if x is np.nan else 1 )<feature_engineering>
sub = pd.read_csv('/kaggle/input/ieee-fraud-detection/sample_submission.csv')
 sub['isFraud'] = clf.predict_proba(test.drop(columns=cols_to_drop)) [:, 1]
sub['TransactionID'] = test.TransactionID
sub.to_csv('submission.csv', index=False )