Upload 5 files

app.py

@@ -0,0 +1,56 @@
+from joblib import load
+import gradio as gr
+import pandas as pd
+import numpy as np
+import shap
+import matplotlib.pyplot as plt
+from features import extract_basic_aggregations
+from os import listdir
+from os.path import join, isfile
+
+examples_path = './csv_examples/'
+examples = [[join(examples_path, f), 'A'] for f in listdir(examples_path)]
+model = load('xgb_cpu.joblib')
+
+explainer = shap.TreeExplainer(model)
+products = {'A': 0,
+            'B': 1,
+            'C': 2,
+            'D': 3,
+            'E': 4
+           }
+def score_client(card_transactions_file, product: str):
+    df = pd.read_csv(card_transactions_file)
+    assert product in products
+
+    features = extract_basic_aggregations(df, cat_columns=['mcc_category', 'day_of_week', 'operation_type'])
+    features = features.reindex(columns=model.feature_names_in_, fill_value=0)
+    features['product'] = products[product]
+    default_proba = model.predict_proba(features)[0][0]
+    shap_values = explainer.shap_values(features)
+    shap.plots.waterfall(explainer(features)[0], max_display=14, show=False)
+    plt.tight_layout()
+    shap_fig = plt.gcf()
+
+    plt.close()
+    return default_proba, shap_fig
+
+
+title = "Check your credit score"
+description = "Check your credit score dude"
+
+inputs = [gr.File(), gr.Dropdown(choices=list(products.keys()), value=list(products.keys())[0])]
+outputs =  [gr.Textbox(label='Your credit score (the more, the better)', interactive=False),
+            gr.Plot(label='SHAP')
+           ]
+
+demo = gr.Interface(
+    fn=score_client,
+    inputs=inputs,
+    outputs=outputs,
+    allow_flagging='never',
+    examples=examples,
+    title=title,
+    description=description,
+)
+demo.launch()

features.py

@@ -0,0 +1,66 @@
+import pandas as pd
+
+CAT_COLUMNS = ['currency', 'operation_kind', 'card_type',
+               'operation_type', 'operation_type_group', 'ecommerce_flag',
+               'payment_system', 'income_flag', 'mcc', 'country', 'city',
+               'mcc_category', 'day_of_week', 'hour','weekofyear']
+
+NUMERIC_COLUMNS = ['days_before', 'hour_diff']
+
+REAL_COLUMNS = ['amnt']
+
+
+def __amnt_pivot_table_by_column_as_frame(frame, column, agg_funcs=None) -> pd.DataFrame:
+    """
+    Generates pivot table for `app_id` and a specified column by aggregating `amnt` column
+
+    :param frame: pd.DataFrame containing card transactions
+    :param column: column with keys to group by on the pivot table column
+    :param agg_funcs: list of aggregation functions, default is ['sum', 'mean', 'count']
+    :return: pd.DataFrame pivot table
+    """
+    if agg_funcs is None:
+        agg_funcs = ['sum', 'mean', 'count']
+    aggs = pd.pivot_table(frame, values='amnt',
+                          index=['app_id'], columns=[column],
+                          aggfunc={'amnt': agg_funcs},
+                          fill_value=0)
+    aggs.columns = [f'amnt_{col[0]}_{column}_{col[1]}' for col in aggs.columns.values]
+    return aggs
+
+
+def extract_basic_aggregations(transactions_frame: pd.DataFrame, cat_columns=None, agg_funcs=None) -> pd.DataFrame:
+    """
+    Extracts basic features from a card transaction dataframe
+
+    :param transactions_frame: pd.DataFrame containing card transactions
+    :param cat_columns: list of categorical columns for which we want to aggregate `amnt`, default is all
+    :param agg_funcs: list of aggregation functions for cat_columns, default is
+    ['sum', 'mean', 'count']
+    :return: pd.DataFrame with extracted features
+    """
+    if not cat_columns:
+        cat_columns = CAT_COLUMNS
+
+    if not agg_funcs:
+        agg_funcs = ['sum', 'mean', 'count']
+
+    pivot_tables = []
+    for col in cat_columns:
+        pivot_tables.append(__amnt_pivot_table_by_column_as_frame(transactions_frame, column=col,
+                                                                  agg_funcs=agg_funcs))
+    pivot_tables = pd.concat(pivot_tables, axis=1)
+
+    # we will also generate total statistics grouped by app_id
+    aggs = {
+        # transation amount
+        'amnt': ['max', 'min', 'mean', 'median', 'sum', 'std'],
+        # time difference between transactions
+        'hour_diff': ['max', 'mean', 'median', 'var', 'std'],
+        # days left before application at the moment when transaction took place
+        'days_before': ['min', 'max', 'median']}
+
+    numeric_stats = transactions_frame.groupby(['app_id']).agg(aggs)
+    numeric_stats.columns = numeric_stats.columns.map('_'.join)
+
+    return pd.concat([pivot_tables, numeric_stats], axis=1).reset_index()

requirements.txt

@@ -0,0 +1,7 @@
+joblib==1.2.0
+gradio==3.12.0
+shap==0.41.0
+matplotlib==3.5.3
+jupytext==1.14.1
+pandas==1.3.5
+tqdm==4.64.0

utils.py

@@ -0,0 +1,38 @@
+import os
+import pandas as pd
+import tqdm
+
+
+def read_parquet_dataset_from_local(path_to_dataset: str, start_from: int = 0,
+                                     num_parts_to_read: int = 1, columns=None, verbose=False) -> pd.DataFrame:
+    """
+    Reads num_parts_to_read parquet partitions and returns the resulting pd.DataFrame
+
+    :param path_to_dataset: directory with parquet partitions
+    :param start_from: partition number to start with
+    :param num_parts_to_read: amount of partitions to read
+    :param columns: columns to read and include
+    :return: pd.DataFrame
+    """
+
+    res = []
+    dataset_paths = sorted([os.path.join(path_to_dataset, filename) for filename in os.listdir(path_to_dataset)
+                              if filename.startswith('part')])
+
+    start_from = max(0, start_from)
+    if num_parts_to_read < 0:
+        chunks = dataset_paths[start_from: ]
+    else:
+        chunks = dataset_paths[start_from: start_from + num_parts_to_read]
+    if verbose:
+        print('Reading chunks:\n')
+        for chunk in chunks:
+            print(chunk)
+    for chunk_path in tqdm.tqdm_notebook(chunks, desc="Reading dataset with pandas"):
+        chunk = pd.read_parquet(chunk_path, columns=columns)
+        for col_name, col_type in [('amnt', 'float32'), ('hour_diff', 'int32')]:
+            if col_name in chunk.columns:
+                chunk[col_name] = chunk[col_name].astype(col_type)
+
+        res.append(chunk)
+    return pd.concat(res).reset_index(drop=True)

xgb_cpu.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:98109adf5d911c8839e61f88484823e7e71554c827b1d73485ef769c470bd39a
+size 2865173