new version of signature model v35

email_parser/_models_signatures.py

@@ -157,10 +157,13 @@ def generate_x_y(df, minmax_scaler=None, standard_scaler=None, n_last_lines_to_k
     df, minmax_scaler, standard_scaler = f_scale_parameters(df, minmax_scaler, standard_scaler)
     x = df[list_columns].to_numpy()[-n_last_lines_to_keep:, :]
     x = np.expand_dims(x, axis=0)
+    x = pad_sequences(x, dtype='float64', value=0, maxlen=n_last_lines_to_keep)
+
     y = df["is_signature"].to_numpy()[-n_last_lines_to_keep:]
     y = np.expand_dims(y, axis=0)
-    return x, y, minmax_scaler, standard_scaler
-
+    y_out = pad_sequences(y, value=0, maxlen=n_last_lines_to_keep)
+    y_mask = pad_sequences(y,  value=-1, maxlen=n_last_lines_to_keep)
+    return x, y_out, y_mask, minmax_scaler, standard_scaler
 
 def f_scale_parameters(df_tagged_data, minmax_scaler=None, standard_scaler=None):
     # df_tagged_data = df_tagged_data.copy(deep=True)

email_parser/config.ini

@@ -3,5 +3,5 @@ ner_model_fr = Jean-Baptiste/camembert-ner-with-dates
 ner_model_en = Jean-Baptiste/roberta-large-ner-english
 device = -1
 default_lang = en
-name_model_signature = model_signature_lstm_v16
+name_model_signature = model_signature_lstm_v35
 path_models = models

email_parser/models/{model_signature_lstm_v16 → model_signature_lstm_v35}/keras_metadata.pb

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:73a9ab3d0262141747b5aafa348c57e1a5813e03cf1e5c291162b37cdf846143
-size 22142
+oid sha256:542eb34419276ac2cf6b8daa50f9d5f8ba25610844db4c38a2ce3986eb393819
+size 22118

email_parser/models/{model_signature_lstm_v16 → model_signature_lstm_v35}/saved_model.pb

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:70ff3069a27b192313bbdd9685f106fc46ef2082eab6479697f575bad555eb4f
-size 5296390
+oid sha256:2c200a10c531b9a7062058f2eb8027d22b9260c652f6d1382322c3d7a73e6248
+size 5294382

email_parser/nlp.py

@@ -285,14 +285,14 @@ def f_detect_email_signature(text, df_ner=None, cut_off_score=0.6, lang=default_
     #     We add dummy value for signature in order to use same function than for training of the model
     df_features["is_signature"] = -2
 
-    x, y_out, _, _ = _models_signatures.generate_x_y(df_features, _models_signatures.minmax_scaler,
+    x, y_out, y_mask, _, _ = _models_signatures.generate_x_y(df_features, _models_signatures.minmax_scaler,
                                                              _models_signatures.standard_scaler)
 
     y_predict = _models_signatures.model.predict(x)
-    y_predict_value = (y_predict> cut_off_score).reshape([-1])
+    y_predict_value = (y_predict[y_mask != -1]> cut_off_score).reshape([-1])
     y_predict_value = np.pad(y_predict_value, (len(df_features) - len(y_predict_value), 0), constant_values=0)[
                       -len(df_features):]
-    y_predict_score = y_predict.reshape([-1])
+    y_predict_score = y_predict[y_mask != -1].reshape([-1])
     y_predict_score = np.pad(y_predict_score, (len(df_features) - len(y_predict_score), 0), constant_values=1)[
                       -len(df_features):]