Repository: EvoTestOps/LogLead (language: python)

Issue:
Refactor loglead files to folders
loader.py, enhancers.py, and anomaly_detection.py to folders loaders, enhancers, anomaly_detectors and separate these big files to separate files.  


Discussion / hints:
Loaders refactored.

Fix patch:
diff --git a/.gitignore b/.gitignore
index a06d0c8..e3992a2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,4 +8,5 @@ private_data/
 *wip/
 .env
 *latex.txt
-*.csv
\ No newline at end of file
+*.csv
+.idea
\ No newline at end of file
diff --git a/demo/HDFS_samples.py b/demo/HDFS_samples.py
index 26b1ac8..f38745a 100644
--- a/demo/HDFS_samples.py
+++ b/demo/HDFS_samples.py
@@ -1,26 +1,29 @@
-#This is an example using HDFS data from samples folder. See also similar TB_samples.py 
-#The files have been loaded from raw and processed to parquet file format for efficient storage.
-#This demonstrates how to work after you have completed the loader.
+# This is an example using HDFS data from samples folder. See also similar TB_samples.py
+# The files have been loaded from raw and processed to parquet file format for efficient storage.
+# This demonstrates how to work after you have completed the loader.
 
-#______________________________________________________________________________
-#Part 1 load libraries and setup paths. 
+# ______________________________________________________________________________
+# Part 1 load libraries and setup paths.
 import sys
 import os
-#Ensure this always gets executed in the same location
+import random
+
+
+# Ensure this always gets executed in the same location
 script_dir = os.path.dirname(os.path.abspath(__file__))
 os.chdir(script_dir)
 sys.path.append('..')
-import loglead.loaders.hdfs as load
-import loglead.enhancer as er
-import loglead.anomaly_detection as ad
 import polars as pl
-import random
-#Location of our sample data
-sample_data="../samples"
 
-#_________________________________________________________________________________
-#Part 2 load data from sample file
-#Load HDFS from sample data
+from loglead import AnomalyDetector
+from loglead.enhancers import EventLogEnhancer, SequenceEnhancer
+
+# Location of our sample data
+sample_data = "../samples"
+
+# _________________________________________________________________________________
+# Part 2 load data from sample file
+# Load HDFS from sample data
 df = pl.read_parquet(f"{sample_data}/hdfs_events_2percent.parquet")
 df_seqs = pl.read_parquet(f"{sample_data}/hdfs_seqs_2percent.parquet")
 print(f"Read HDFS 2% sample. Numbers of events is: {len(df)} and number of sequences is {len(df_seqs)}")
@@ -28,21 +31,24 @@
 print(f"Anomaly count {ano_count}. Anomaly percentage in Sequences {ano_count/len(df_seqs)*100:.2f}%")
 
 
-#_________________________________________________________________________________
-#Part 3 add enhanced reprisentations 
+# _________________________________________________________________________________
+# Part 3 add enhanced reprisentations
 print(f"\nStarting enhancing all log events:")
-enhancer = er.EventLogEnhancer(df)
+enhancer = EventLogEnhancer(df)
+
 
 # For nicer printing a function to format series of elements as a list-like string
 def format_as_list(series):
     elements = [str(item) for item in series]
     return '[' + ', '.join(elements) + ']'
-#Pick a random line
+
+
+# Pick a random line
 row_index = random.randint(0, len(df) - 1)
 
 
 print(f"Original log message: {df['m_message'][row_index]}")
-#Create some enhanced representations
+# Create some enhanced representations
 df = enhancer.normalize()
 print(f"as normalized:        {df['e_message_normalized'][row_index]}")
 df = enhancer.words()
@@ -69,15 +75,14 @@ def format_as_list(series):
 print(f"event length words:   {df['e_words_len'][row_index]}")
 
 
-
-#_________________________________________________________________________________
-#Part 4 Aggregate event level data to sequence level
+# _________________________________________________________________________________
+# Part 4 Aggregate event level data to sequence level
 print(f"\nStarting aggregating log event info to log sequences:")
 
 seqs_row_index = random.randint(0, len(df_seqs) - 1)
 seq_id = df_seqs['seq_id'][seqs_row_index]
 print(f"Sequence level dataframe without aggregated info: {df_seqs.filter(pl.col('seq_id') == seq_id)}")
-seq_enhancer = er.SequenceEnhancer(df = df, df_seq = df_seqs)
+seq_enhancer = SequenceEnhancer(df=df, df_seq=df_seqs)
 df_seqs = seq_enhancer.events("e_event_tip_id")
 df_seqs = seq_enhancer.events("e_event_pliplom_id")
 print(f"list of Tipping events in a sequence: {format_as_list(df_seqs.filter(pl.col('seq_id') == seq_id)['e_event_tip_id'][0])}")
@@ -94,64 +99,64 @@ def format_as_list(series):
 df_seqs = seq_enhancer.tokens(token="e_words")
 print(f"Sequence level dataframe with aggregated info: {df_seqs.filter(pl.col('seq_id') == seq_id)}")
 
-#_________________________________________________________________________________________
-#Part 5 Do some anomaly detection
+# _________________________________________________________________________________________
+# Part 5 Do some anomaly detection
 print(f"\nStarting anomaly detection of HDFS Sequences")
 print(f"Predicting with sequence length and duration ")
 numeric_cols = ["seq_len",  "duration_sec",]
-#Auc-roc computation is not enabled by default as it can reduce speed
-sad = ad.AnomalyDetection(auc_roc = True) 
-#Using 10% for training 90% for testing
+# Auc-roc computation is not enabled by default as it can reduce speed
+sad = AnomalyDetector(auc_roc=True)
+# Using 10% for training 90% for testing
 sad.numeric_cols = numeric_cols
-sad.test_train_split (seq_enhancer.df_seq, test_frac=0.90)
+sad.test_train_split(seq_enhancer.df_seq, test_frac=0.90)
 print(f"using 10% for training and 90% for testing")
 
-#Logistic Regression
+# Logistic Regression
 sad.train_LR()
 df_seq = sad.predict()
-#Use Decision Tree
+# Use Decision Tree
 sad.train_DT()
 df_seq = sad.predict()
 
 print(f"Predicting with words")
 sad.item_list_col = "e_words"
-sad.numeric_cols = None #Important otherwise we use both numeric_col and item_list_col for predicting
-sad.prepare_train_test_data() #Data needs to prepared after changing the predictor columns
-#Logistic Regression
+sad.numeric_cols = None  # Important otherwise we use both numeric_col and item_list_col for predicting
+sad.prepare_train_test_data()  # Data needs to prepared after changing the predictor columns
+# Logistic Regression
 sad.train_LR()
 df_seq = sad.predict()
-#Use Decision Tree
+# Use Decision Tree
 sad.train_DT()
 df_seq = sad.predict()
 
 print(f"Predicting with PL-Iplom parsing results")
 sad.item_list_col = "e_event_pliplom_id"
 sad.prepare_train_test_data()
-#Logistic Regression
+# Logistic Regression
 sad.train_LR()
 df_seq = sad.predict()
-#Use Decision Tree
+# Use Decision Tree
 sad.train_DT()
 df_seq = sad.predict()
 
-#____________________________________________________________
-#Part 6 run all anomaly detectors and store score to Pandas table for easy storage
+# ____________________________________________________________
+# Part 6 run all anomaly detectors and store score to Pandas table for easy storage
 print(f"Running all anomaly detectors with Words and Trigrams and storing results")
 print(f"We run everything two times - Adjust as needed")
 
-sad = ad.AnomalyDetection(store_scores=True, print_scores=False, auc_roc = True)
-for i in range(3): #We do just three loops in this demo
+sad = AnomalyDetector(store_scores=True, print_scores=False, auc_roc=True)
+for i in range(3):  # We do just three loops in this demo
     sad.item_list_col = "e_words"
-    sad.test_train_split (seq_enhancer.df_seq, test_frac=0.90)
+    sad.test_train_split(seq_enhancer.df_seq, test_frac=0.90)
     sad.evaluate_all_ads()
     
-    #We keep existing split but need to prepare everything with trigrams
+    # We keep existing split but need to prepare everything with trigrams
     sad.item_list_col = "e_trigrams"
     sad.prepare_train_test_data()
     sad.evaluate_all_ads()
 
 print(f"Inspecting results...") 
-print ("Average of accuracy:")
+print("Average of accuracy:")
 print(sad.storage.calculate_average_scores(score_type="accuracy", metric='mean'))
 print(f"Median of F1:")
 print(sad.storage.calculate_average_scores(score_type="f1", metric='median'))
@@ -164,6 +169,4 @@ def format_as_list(series):
 #print(sad.storage.calculate_average_scores(score_type="f1", metric='median').to_latex())
 
 print(f"Confusion matrixes can also be inspected")
-sad.storage.print_confusion_matrices("LogisticRegression","e_trigrams")
-
-
+sad.storage.print_confusion_matrices("LogisticRegression", "e_trigrams")
diff --git a/demo/TB_samples.py b/demo/TB_samples.py
index d9cdacf..2f59ab9 100644
--- a/demo/TB_samples.py
+++ b/demo/TB_samples.py
@@ -1,47 +1,53 @@
-#This is an example using TB (Thunderbird) data from samples folder. See also similar HDFS_samples.py  
-#The files have been loaded from raw and processed to parquet file format for efficient storage.
-#This file demonstrates how to work after you have completed the loader.
+# This is an example using TB (Thunderbird) data from samples folder. See also similar HDFS_samples.py
+# The files have been loaded from raw and processed to parquet file format for efficient storage.
+# This file demonstrates how to work after you have completed the loader.
 
-#______________________________________________________________________________
-#Part 1 load libraries and setup paths. 
+# ______________________________________________________________________________
+# Part 1 load libraries and setup paths.
 import sys
 import os
-#Ensure this always gets executed in the same location
+
+import polars as pl
+
+# Ensure this always gets executed in the same location
 script_dir = os.path.dirname(os.path.abspath(__file__))
 os.chdir(script_dir)
 sys.path.append('..')
-import loglead.loaders.base as load
-import loglead.enhancer as er
-import loglead.anomaly_detection as ad
-import polars as pl
-import random
-#Location of our sample data
-sample_data="../samples"
 
-#_________________________________________________________________________________
-#Part 2 load data from sample file
-#Load TB from sample data
+from loglead.enhancers import EventLogEnhancer
+from loglead import AnomalyDetector
+
+# Location of our sample data
+sample_data = "../samples"
+
+# _________________________________________________________________________________
+# Part 2 load data from sample file
+# Load TB from sample data
 df = pl.read_parquet(f"{sample_data}/tb_0125percent.parquet")
 print(f"Read TB 0.125% sample. Numbers of events: {len(df)}")
 ano_count = df["anomaly"].sum()
 print(f"Anomaly count {ano_count}. Anomaly percentage in Events {ano_count/len(df)*100:.2f}%")
 
 
-#_________________________________________________________________________________
-#Part 3 add enhanced reprisentations 
+# _________________________________________________________________________________
+# Part 3 add enhanced representations
 print(f"\nStarting enhancing all log events:")
-enhancer = er.EventLogEnhancer(df)
+enhancer = EventLogEnhancer(df)
+
 
 # For nicer printing a function to format series of elements as a list-like string
 def format_as_list(series):
     elements = [str(item) for item in series]
     return '[' + ', '.join(elements) + ']'
-#Pick a random line
-row_index = 1 #random.randint(0, len(df) - 1)
+
+
+# Pick a random line
+row_index = 1
+# row_index = random.randint(0, len(df) - 1)
 
 
 print(f"Original log message: {df['m_message'][row_index]}")
-#Create some enhanced representations
+# Create some enhanced representations
 df = enhancer.normalize()
 print(f"as normalized:        {df['e_message_normalized'][row_index]}")
 df = enhancer.words()
@@ -52,65 +58,65 @@ def format_as_list(series):
 print(f"as Drain event id:    {df['e_event_drain_id'][row_index]}")
 df = enhancer.parse_tip()
 print(f"as Tipping event id:    {df['e_event_tip_id'][row_index]}")
-#Spell parser takes a bit too long for the video  
-#df = enhancer.parse_spell()
-#print(f"as Spell event id:    {df['e_event_spell_id'][row_index]}")
+# Spell parser takes a bit too long for the video
+# df = enhancer.parse_spell()
+# print(f"as Spell event id:    {df['e_event_spell_id'][row_index]}")
 df = enhancer.length()
 print(f"event length chars:   {df['e_chars_len'][row_index]}")
 print(f"event length lines:   {df['e_lines_len'][row_index]}")
 print(f"event length words:   {df['e_words_len'][row_index]}")
 
-#_________________________________________________________________________________________
-#Part 5 we do some anomaly detection. No part 4 here as TB is not labeled on sequence level. See HDFS_samples.py
+# _________________________________________________________________________________________
+# Part 5 we do some anomaly detection. No part 4 here as TB is not labeled on sequence level. See HDFS_samples.py
 print(f"\nStarting anomaly detection of TB Events")
 numeric_cols = ["e_chars_len",  "e_lines_len",]
-sad = ad.AnomalyDetection()
-#Using 10% for training 90% for testing
+sad = AnomalyDetector()
+# Using 10% for training 90% for testing
 sad.numeric_cols = numeric_cols
-sad.test_train_split (df, test_frac=0.90)
+sad.test_train_split(df, test_frac=0.90)
 print(f"using 10% for training and 90% for testing")
 print(f"Predicting with sequence length and duration ")
 
-#Logistic Regression
+# Logistic Regression
 sad.train_LR()
 df_seq = sad.predict()
-#Use Decision Tree
+# Use Decision Tree
 sad.train_DT()
 df_seq = sad.predict()
 
 print(f"Predicting with words")
 sad.item_list_col = "e_words"
-sad.numeric_cols = None #Important otherwise we use both numeric_col and item_list_col for predicting
-sad.prepare_train_test_data() #Data needs to prepared after changing the predictor columns
-#Logistic Regression
+sad.numeric_cols = None  # Important otherwise we use both numeric_col and item_list_col for predicting
+sad.prepare_train_test_data()  # Data needs to prepared after changing the predictor columns
+# Logistic Regression
 sad.train_LR()
 df_seq = sad.predict()
-#Use Decision Tree
+# Use Decision Tree
 sad.train_DT()
 df_seq = sad.predict()
 
 print(f"Predicting with Drain parsing results")
 sad.item_list_col = "e_event_drain_id"
 sad.prepare_train_test_data()
-#Logistic Regression
+# Logistic Regression
 sad.train_LR()
 df_seq = sad.predict()
-#Use Decision Tree
+# Use Decision Tree
 sad.train_DT()
 df_seq = sad.predict()
 
-#____________________________________________________________
-#Part 6 run all anomaly detectors and store scores
+# ____________________________________________________________
+# Part 6 run all anomaly detectors and store scores
 print(f"Running all anomaly detectors with Words and Trigrams and storing results")
 print(f"We run everything two times - Adjust as needed")
 
-sad = ad.AnomalyDetection(store_scores=True, print_scores=False)
-for i in range(2): #We do just two loops in this demo
+sad = AnomalyDetector(store_scores=True, print_scores=False)
+for i in range(2):  # We do just two loops in this demo
     sad.item_list_col = "e_words"
-    sad.test_train_split (df, test_frac=0.90)
+    sad.test_train_split(df, test_frac=0.90)
     sad.evaluate_all_ads()
     
-    #We keep existing split but need to prepare data for trigram data
+    # We keep existing split but need to prepare data for trigram data
     sad.item_list_col = "e_trigrams"
     sad.prepare_train_test_data()
     sad.evaluate_all_ads()
@@ -118,8 +124,5 @@ def format_as_list(series):
 
 print(f"Inspecting results. Averages of runs:")
 print(sad.storage.calculate_average_scores(score_type="accuracy").to_csv())
-print(f"Confusion matrixes can also be inspected")
-sad.storage.print_confusion_matrices("LogisticRegression","e_trigrams")
-
-
-
+print(f"Confusion matrices can also be inspected")
+sad.storage.print_confusion_matrices("LogisticRegression", "e_trigrams")
diff --git a/demo/parser_benchmark/ano_detection.py b/demo/parser_benchmark/ano_detection.py
index a2a390a..67b18ec 100644
--- a/demo/parser_benchmark/ano_detection.py
+++ b/demo/parser_benchmark/ano_detection.py
@@ -1,21 +1,23 @@
 import gc
 import time
-import yaml
-import sys
-import polars as pl
 import os
+import sys
+import argparse
+import datetime
+import random
+
 print(os.getcwd())
 from dotenv import load_dotenv, find_dotenv
 load_dotenv(find_dotenv())
-import sys
+import polars as pl
+import yaml
 LOGLEAD_PATH = os.environ.get("LOGLEAD_PATH")
 sys.path.append(os.environ.get("LOGLEAD_PATH"))
-import loglead.loaders as loaders 
-import loglead.enhancer as er
-import loglead.anomaly_detection as ad
-import argparse
-import datetime
-import random
+
+from loglead import AnomalyDetector
+from loglead.loaders import *
+from loglead.enhancers import EventLogEnhancer, SequenceEnhancer
+
 
 #import warnings
 #from sklearn.exceptions import ConvergenceWarning
@@ -33,20 +35,24 @@
                     help='Configuration file path')
 args = parser.parse_args()
 
+
 def load_config(config_path):
     with open(config_path, 'r') as file:
         return yaml.safe_load(file)
 
+
 config = load_config(args.config_path)
 
+
 def proportion_to_float(proportion_str):
     numerator, denominator = map(int, proportion_str.split('/'))
     return numerator / denominator
 
 
-def print_latex_table(storage, data_name,data_proportion,redraws, metric, score_type, repeats, chrono_order, normalize, rows, anos, test_frac):
+def print_latex_table(storage, data_name, data_proportion, redraws, metric, score_type, repeats, chrono_order,
+                      normalize, rows, anos, test_frac):
     # Print the modified LaTeX table
-        # Get the current timestamp and format it as a string
+    # Get the current timestamp and format it as a string
     timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
     # Create the filename using the timestamp
     filename = f"Table_{data_name}_{timestamp}_latex.txt"
@@ -67,6 +73,7 @@ def print_latex_table(storage, data_name,data_proportion,redraws, metric, score_
                 # Print a message indicating that the table was written to the file
     print(f"LaTeX tables written to {filename}")
 
+
 def handle_test_fraction(test_fraction):
     values_to_process = []
 
@@ -76,7 +83,7 @@ def handle_test_fraction(test_fraction):
         max_value = test_fraction['max']
         step = test_fraction['step']
         current_value = min_value
-        while current_value < max_value + 0.001:#sometimes values read from disk or added come as 0.9499999 when desired  is 0.95
+        while current_value < max_value + 0.001:  #sometimes values read from disk or added come as 0.9499999 when desired  is 0.95
             rounded_value = round(current_value, 2)
             values_to_process.append(rounded_value)
             current_value += step
@@ -89,6 +96,7 @@ def handle_test_fraction(test_fraction):
 
     return values_to_process
 
+
 for dataset_name, dataset_info in config['datasets'].items():
     gc.collect()
     print(f"_________________________________________________________________________")
@@ -102,7 +110,7 @@ def handle_test_fraction(test_fraction):
     data_normalize = dataset_info['normalize']
  
     # Dynamic dataset loading based on configuration
-    loader_class = getattr(loaders, dataset_info['loader'])
+    loader_class = eval(f"{dataset_info['loader']}")
     loader_args = {key: full_data + value if key.endswith('name') else value 
                    for key, value in dataset_info.items() if key == 'filename' or key =='filename_pattern' or key =='labels_file_name'}
     loader = loader_class(**loader_args)
@@ -110,14 +118,14 @@ def handle_test_fraction(test_fraction):
     loader.execute()
     time_elapsed = time.time() - time_start
     print(f'Data {dataset_name} loaded in {time_elapsed:.2f} seconds')
-    #In Hadoop anomaly is majority class. Flip it to keep F1-binary equally difficult.
+    # In Hadoop anomaly is majority class. Flip it to keep F1-binary equally difficult.
     if dataset_name == "Hadoop":
-       loader.df_seq = loader.df_seq.with_columns(~pl.col("anomaly"))
-    sad = ad.AnomalyDetection(store_scores=True, print_scores=False, auc_roc=True)
-    for i in range (data_redraws):
+        loader.df_seq = loader.df_seq.with_columns(~pl.col("anomaly"))
+    sad = AnomalyDetector(store_scores=True, print_scores=False, auc_roc=True)
+    for i in range(data_redraws):
         proportion = proportion_to_float(data_proportion)
-        if proportion < 1: #Create a new baseloader for reducing dataframes. 
-            loader_redu = loaders.BaseLoader(filename = "Dummy", df = loader.df, df_seq = loader.df_seq) 
+        if proportion < 1:  # Create a new baseloader for reducing dataframes.
+            loader_redu = BaseLoader(filename="Dummy", df=loader.df, df_seq=loader.df_seq)
             loader_redu.reduce_dataframes(frac=proportion,  random_state=random.randint(0, 100000))
         else:
             loader_redu = loader
@@ -129,10 +137,10 @@ def handle_test_fraction(test_fraction):
         #Kill nulls if they still exist
         df = df.filter(pl.col("m_message").is_not_null())
 
-        enhancer = er.EventLogEnhancer(df)
+        enhancer = EventLogEnhancer(df)
         time_start = time.time()
         parser_params = None
-        #In parsing do we normalize the input with regular expression or no
+        # In parsing do we normalize the input with regular expression or no
         if data_normalize:
             df = enhancer.normalize()
             df = enhancer.words(column="e_message_normalized")
@@ -144,24 +152,24 @@ def handle_test_fraction(test_fraction):
         print(f'Data normalized and split to words redraws:{i+1}/{data_redraws}, proportion:{data_proportion}, rows:{df.height}, time {time_elapsed:.2f} seconds')
         import logging
         logging.basicConfig(level=logging.ERROR,
-                        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
-                        datefmt='%Y-%m-%d %H:%M:%S')
+                            format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+                            datefmt='%Y-%m-%d %H:%M:%S')
         logger = logging.getLogger(__name__)
-        #Parse all for this dataset
+        # Parse all for this dataset
         for parser in config['parsers']:
             parser_name = parser['name']
             parser_call = parser['call']
             parser_field = parser['field']
             # Parse
-            enhancer = er.EventLogEnhancer(df)
+            enhancer = EventLogEnhancer(df)
             time_start = time.time()
             method_to_call = getattr(enhancer, parser_call)
             df = method_to_call(**params)
             time_elapsed = time.time() - time_start
             print(f'Data parsed parser:{parser_name}, rows:{df.height}, time:{time_elapsed}s')
-            #Aggregate to sequences
+            # Aggregate to sequences
             if data_predict == 'seq':
-                seq_enhancer = er.SequenceEnhancer(enhancer.df, df_seq)
+                seq_enhancer = SequenceEnhancer(enhancer.df, df_seq)
                 df_seq = seq_enhancer.events(parser_field)
                 df_to_predict = df_seq
             else:
@@ -170,8 +178,8 @@ def handle_test_fraction(test_fraction):
         #print (f"Columns in df_to_predict: {df_to_predict.columns}")
         print(f'Predicting repeats:{data_repeats} data_rows:{df_to_predict.height}, anomalies:{df_to_predict["anomaly"].sum()} fraction for testing:{data_test_fraction}, chrono order:{data_chrono_order}')    
         for tf in data_test_fraction:
-            print (f'Test fraction:{tf}')
-            for j in range (data_repeats):
+            print(f'Test fraction:{tf}')
+            for j in range(data_repeats):
                 print(f"Run:{j}", end=" ")
                 time_start = time.time()
                 first = True
@@ -181,16 +189,16 @@ def handle_test_fraction(test_fraction):
                     print(f"Parser:{parser_name}", end=" ")
                     sad.item_list_col = parser_field
                     if first:
-                        sad.test_train_split (df_to_predict, test_frac=tf, shuffle=not data_chrono_order)
+                        sad.test_train_split(df_to_predict, test_frac=tf, shuffle=not data_chrono_order)
                         first = False
-                    else: # We keep the existing split but prepare a new parserfield
+                    else:  # We keep the existing split but prepare a new parserfield
                         sad.prepare_train_test_data()
                     for algo in config['algos']:
                         algo_call = algo['call']
                         algo_name = algo['name']
-                        print(f".",end="")
+                        print(f".", end="")
                         sys.stdout.flush()
-                        #sad = ad.AnomalyDetection(store_scores=True, print_scores=False)
+                        #sad = AnomalyDetection(store_scores=True, print_scores=False)
                         time_algo_start = time.time()
                         method_to_call = getattr(sad, algo_call)
                         result = method_to_call()
@@ -201,7 +209,8 @@ def handle_test_fraction(test_fraction):
                 time_elapsed = time.time() - time_start
                 print(f' time:{time_elapsed}')
 
-    print_latex_table(sad.storage, dataset_name,data_proportion, score_type=["auc-roc", "f1"], metric=["median", "mean"], 
-                      redraws = data_redraws, repeats= data_repeats, chrono_order= data_chrono_order, normalize=data_normalize,
-                      rows = df.height,anos = df_to_predict["anomaly"].sum(), test_frac= data_test_fraction)
+    print_latex_table(sad.storage, dataset_name, data_proportion, score_type=["auc-roc", "f1"],
+                      metric=["median", "mean"], redraws=data_redraws, repeats=data_repeats,
+                      chrono_order=data_chrono_order, normalize=data_normalize, rows=df.height,
+                      anos=df_to_predict["anomaly"].sum(), test_frac=data_test_fraction)
            
\ No newline at end of file
diff --git a/demo/parser_benchmark/parsing_speed.py b/demo/parser_benchmark/parsing_speed.py
index 9dd9398..9dcfb9b 100644
--- a/demo/parser_benchmark/parsing_speed.py
+++ b/demo/parser_benchmark/parsing_speed.py
@@ -1,26 +1,30 @@
 import gc
 import time
+import argparse
+
 import yaml
-import sys
 import polars as pl
-import os
-print(os.getcwd())
-#sys.path.append('../..')
 import sys
-sys.path.append('/home/mmantyla/LogLead')
-import loglead.loaders as loaders  # Assuming loaders are correctly organized
-#from loglead.loaders.hadoop import HadoopLoader
-import loglead.enhancer as er
-import loglead.anomaly_detection as ad
-import argparse
+import os
+from dotenv import load_dotenv, find_dotenv
+load_dotenv(find_dotenv())
+LOGLEAD_PATH = os.environ.get("LOGLEAD_PATH")
+sys.path.append(os.environ.get("LOGLEAD_PATH"))
+print(LOGLEAD_PATH)
+from loglead.loaders import *
+from loglead.enhancers import EventLogEnhancer
 
 # Base path for datasets
 full_data = "/home/mmantyla/Datasets"
 
 # Load the configuration
-default_config = '/home/mmantyla/LogLead/demo/parser_benchmark/speed_config.yml'
+#default_config = '/home/mmantyla/LogLead/demo/parser_benchmark/speed_config.yml'
+# Base path for datasets
+full_data = os.environ.get("LOG_DATA_PATH")
+# Load the configuration
+default_config = LOGLEAD_PATH + '/demo/parser_benchmark/speed_config.yml'
 #default_config = '/home/mmantyla/LogLead/demo/parser_speed_tests/config_missing.yml'
-default_threshold = 600 #How many seconds is the threshold after which remaining runs for the parser are skipped
+default_threshold = 600  # How many seconds is the threshold after which remaining runs for the parser are skipped
 #config_path = '/home/mmantyla/LogLead/demo/parser_speed_tests/config_fiplom.yml'
 
 # Adding argparse for command-line argument parsing
@@ -32,17 +36,19 @@
 args = parser.parse_args()
 
 
-
 def load_config(config_path):
     with open(config_path, 'r') as file:
         return yaml.safe_load(file)
 
+
 config = load_config(args.config_path)
 
+
 def proportion_to_float(proportion_str):
     numerator, denominator = map(int, proportion_str.split('/'))
     return numerator / denominator
 
+
 def is_excluded(parser, dataset, proportion, exclusions):
     # Check if the proportion is a string and convert it to a float if necessary
     if isinstance(proportion, str):
@@ -64,7 +70,6 @@ def is_excluded(parser, dataset, proportion, exclusions):
 # Example usage remains the same
 
 
-
 # Main processing loop
 for dataset_name, dataset_info in config['datasets'].items():
     gc.collect()
@@ -72,9 +77,10 @@ def is_excluded(parser, dataset, proportion, exclusions):
     print(f"Processing dataset: {dataset_name}")
     
     # Dynamic dataset loading based on configuration
-    loader_class = getattr(loaders, dataset_info['loader'])
+    loader_class = eval(f"{dataset_info['loader']}")
     loader_args = {key: full_data + value if key.endswith('name') else value 
                    for key, value in dataset_info.items() if key != 'loader'}
+    print (f"Loader: {loader_class}, args:{loader_args}")
     loader = loader_class(**loader_args)
     time_start = time.time()
 
@@ -93,7 +99,7 @@ def is_excluded(parser, dataset, proportion, exclusions):
     #    df = df.filter(pl.col("system_name") == "TrainTicket")
     df = df.select(pl.col("m_message"))
     
-    enhancer = er.EventLogEnhancer(df)
+    enhancer = EventLogEnhancer(df)
     time_start = time.time()
     df = enhancer.normalize()
     df = enhancer.words(column="e_message_normalized")
@@ -101,11 +107,10 @@ def is_excluded(parser, dataset, proportion, exclusions):
     print(f'Data normalized and split to words {time_elapsed:.2f} seconds')
     import logging
     logging.basicConfig(level=logging.ERROR,
-                    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
-                    datefmt='%Y-%m-%d %H:%M:%S')
+                        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+                        datefmt='%Y-%m-%d %H:%M:%S')
     logger = logging.getLogger(__name__)
 
-
     for parser in config['parsers']:
         parser_name = parser['name']
         parser_call = parser['call']
@@ -119,12 +124,12 @@ def is_excluded(parser, dataset, proportion, exclusions):
             
             # Reduce the dataframe according to the current fraction
             # gc.collect() might be necessary if memory management is a concern
-            loader_class = getattr(loaders, "BaseLoader")
+            loader_class = BaseLoader
             loader2 = loader_class(df=df, filename="Fake")
             df_reduced = loader2.reduce_dataframes(frac=proportion)
             
             # Parse
-            enhancer_reduced = er.EventLogEnhancer(df_reduced)
+            enhancer_reduced = EventLogEnhancer(df_reduced)
             time_start = time.time()
             
             method_to_call = getattr(enhancer_reduced, parser_call)
@@ -146,9 +151,6 @@ def is_excluded(parser, dataset, proportion, exclusions):
             #templates_str = ", ".join(map(str, templates_to_show))  # Convert each template to string in case they are not.
             #print(f'Templates: {templates_str}')
     
-
-
-
         # for proportion_str in config['proportions']:
         #     proportion = proportion_to_float(proportion_str)
             
diff --git a/demo/parser_benchmark/speed_config.yml b/demo/parser_benchmark/speed_config.yml
index c5cf940..d389c10 100644
--- a/demo/parser_benchmark/speed_config.yml
+++ b/demo/parser_benchmark/speed_config.yml
@@ -55,38 +55,38 @@ proportions:
  - 1/1024
  - 1/512
  - 1/256
- - 1/128
- - 1/64
- - 1/32
- - 1/16
- - 1/8
- - 1/4
- - 1/2
- - 1/1
+#  - 1/128
+#  - 1/64
+#  - 1/32
+#  - 1/16
+#  - 1/8
+#  - 1/4
+#  - 1/2
+#  - 1/1
 
 parsers:
-  # - name: Tip
-  #   call: parse_tip
-  #   field: e_event_tip_id
-  # - name: Pliplom
-  #   call: parse_pliplom
-  #   field: e_event_pliplom_id
-  # - name: AEL
-  #   call: parse_ael
-  #   field: e_event_ael_id
-  # - name: Iplom
-  #   call: parse_iplom
-  #   field: e_event_iplom_id
-  # - name: Drain
-  #   call: parse_drain
-  #   field: e_event_drain_id
-  # - name: Brain
-  #   call: parse_brain
-  #   field: e_event_brain_id  
-  - name: Spell
-    call: parse_spell
-    field: e_event_spell_id
-  - name: Lenma
-    call: parse_lenma
-    field: e_event_lenma_id
+  - name: Tip
+    call: parse_tip
+    field: e_event_tip_id
+  - name: Pliplom
+    call: parse_pliplom
+    field: e_event_pliplom_id
+  - name: AEL
+    call: parse_ael
+    field: e_event_ael_id
+  - name: Iplom
+    call: parse_iplom
+    field: e_event_iplom_id
+  - name: Drain
+    call: parse_drain
+    field: e_event_drain_id
+  - name: Brain
+    call: parse_brain
+    field: e_event_brain_id  
+  # - name: Spell
+  #   call: parse_spell
+  #   field: e_event_spell_id
+  # - name: Lenma
+  #   call: parse_lenma
+  #   field: e_event_lenma_id
 
diff --git a/demo/saner_2024_paper/t1_load.py b/demo/saner_2024_paper/t1_load.py
index 9f39fa4..91ab01e 100644
--- a/demo/saner_2024_paper/t1_load.py
+++ b/demo/saner_2024_paper/t1_load.py
@@ -1,44 +1,47 @@
-#This file performs same actions in both LogLEAD and in LogParsers code. 
+# This file performs same actions in both LogLEAD and in LogParsers code.
 # LogParsers loading routine has been slightly modified. Modifications are explained in the separate file
  
-#Separate demo files
+# Separate demo files
 import sys
 import time
 sys.path.append('..')
+
 import polars as pl
 
-import loglead.loaders.base as load
-import demo.paper.t1_logparser_load as logparser
+from loglead.loaders import *
+from .t1_logparser_load import log_to_dataframe
 
 # Base directory for datasets
 #full_data = "/home/ubuntu/Datasets"
 full_data = "/home/mmantyla/Datasets"
 
 logparser_format = {
-    "hdfs" : "<Date> <Time> <Pid> <Level> <Component>: <Content>",
-    "bgl" : "<Label> <Timestamp> <Date> <Node> <Time> <NodeRepeat> <Type> <Component> <Level> <Content>",
-#    "tb" : "<Label> <Timestamp> <Date> <User> <Month> <Day> <Time> <Location> <Component>(\[<PID>\])?: <Content>"
+    "hdfs": "<Date> <Time> <Pid> <Level> <Component>: <Content>",
+    "bgl": "<Label> <Timestamp> <Date> <Node> <Time> <NodeRepeat> <Type> <Component> <Level> <Content>",
+    # "tb" : "<Label> <Timestamp> <Date> <User> <Month> <Day> <Time> <Location> <Component>(\[<PID>\])?: <Content>"
 }
 
 loglead_format = {
-    "hdfs" : ["date", "time", "id", "level", "component", "m_message"],
-    "bgl" : ["label", "timestamp", "date", "node", "time", "noderepeat", "type", "component", "level", "m_message"],
-#    "tb" : ["label", "timestamp", "date", "userid", "month", "day", "time", "location", "component_pid", "m_message"],
-    "hadoop" : ["date", "time","level", "component","m_message"]
+    "hdfs": ["date", "time", "id", "level", "component", "m_message"],
+    "bgl": ["label", "timestamp", "date", "node", "time", "noderepeat", "type", "component", "level", "m_message"],
+    # "tb" : ["label", "timestamp", "date", "userid", "month", "day", "time", "location", "component_pid", "m_message"],
+    "hadoop": ["date", "time", "level", "component", "m_message"]
 }
 
+
 def create_correct_loader(dataset):
-    if dataset=="hdfs":
-        loader = load.HDFSLoader(filename=f"{full_data}/hdfs/HDFS.log", 
-                                            labels_file_name=f"{full_data}/hdfs/anomaly_label.csv")
-    elif dataset=="tb":
-        loader = load.ThunderbirdLoader(filename=f"{full_data}/thunderbird/Thunderbird.log") #Might take 2-3 minutes in HPC cloud. In desktop out of memory
-    elif dataset=="bgl":
-        loader = load.BGLLoader(filename=f"{full_data}/bgl/BGL.log")
-    elif dataset=="hadoop":
-        loader = load.HadoopLoader(filename=f"{full_data}/hadoop/",
-                                                 filename_pattern  ="*.log",
-                                                 labels_file_name=f"{full_data}/hadoop/abnormal_label_accurate.txt") 
+    if dataset == "hdfs":
+        loader = HDFSLoader(filename=f"{full_data}/hdfs/HDFS.log",
+                            labels_file_name=f"{full_data}/hdfs/anomaly_label.csv")
+    elif dataset == "tb":
+        # Might take 2-3 minutes in HPC cloud. In desktop out of memory
+        loader = ThuSpiLibLoader(filename=f"{full_data}/thunderbird/Thunderbird.log")
+    elif dataset == "bgl":
+        loader = BGLLoader(filename=f"{full_data}/bgl/BGL.log")
+    elif dataset == "hadoop":
+        loader = HadoopLoader(filename=f"{full_data}/hadoop/",
+                              filename_pattern="*.log",
+                              labels_file_name=f"{full_data}/hadoop/abnormal_label_accurate.txt")
     return loader
 
 
@@ -50,18 +53,18 @@ def create_correct_loader(dataset):
         time_start = time.time()
         loader = create_correct_loader(key)
         loader.load()
-        if (key == "hadoop"):
+        if key == "hadoop":
             loader._merge_multiline_entries()
             loader._extract_process()
-            #Store columns 
-            df_store_cols = loader.df.select(pl.col("seq_id","seq_id_sub", "process","row_nr", "column_1"))
-            #This method splits the string and overwrite self.df
-            loader._split_and_unnest(["date", "time","level", "component","m_message"])
-            #Merge the stored columns back to self.df
+            # Store columns
+            df_store_cols = loader.df.select(pl.col("seq_id", "seq_id_sub", "process", "row_nr", "column_1"))
+            # This method splits the string and overwrite self.df
+            loader._split_and_unnest(["date", "time", "level", "component", "m_message"])
+            # Merge the stored columns back to self.df
             loader.df = pl.concat([loader.df, df_store_cols], how="horizontal")
         else:    
             loader._split_and_unnest(value)
-        if (key == "tb"):
+        if key == "tb":
             loader._split_component_and_pid()
         loglead_times.append(time.time() - time_start)
 
@@ -75,10 +78,8 @@ def create_correct_loader(dataset):
     for _ in range(1):
         loader = create_correct_loader(key)
         time_start = time.time()
-        df = logparser.log_to_dataframe(loader.filename, value)
+        df = log_to_dataframe(loader.filename, value)
         logparser_times.append(time.time() - time_start)
 
     avg_logparser_time = sum(logparser_times) / len(logparser_times)
     print(f'LogParser {key} load average time over 10 runs: {avg_logparser_time:.2f} seconds.{df.shape[0]} rows processed')
-          
-
diff --git a/demo/saner_2024_paper/t2_drain_parsing.py b/demo/saner_2024_paper/t2_drain_parsing.py
index b7efd21..6460c86 100644
--- a/demo/saner_2024_paper/t2_drain_parsing.py
+++ b/demo/saner_2024_paper/t2_drain_parsing.py
@@ -4,28 +4,28 @@
 import sys
 import time
 sys.path.append('..')
-import polars as pl
 
-import loglead.loaders.base as load
-import loglead.enhancer as er
+from loglead.enhancers import EventLogEnhancer
+from loglead.loaders import *
 
 #full_data = "/home/ubuntu/Datasets"
 full_data = "/home/mmantyla/Datasets"
 
 datasets = ["hdfs", "hadoop", "bgl"]
 
+
 def create_correct_loader(dataset):
-    if dataset=="hdfs":
-        loader = load.HDFSLoader(filename=f"{full_data}/hdfs/HDFS.log", 
-                                            labels_file_name=f"{full_data}/hdfs/anomaly_label.csv")
-    elif dataset=="tb":
-        loader = load.ThunderbirdLoader(filename=f"{full_data}/thunderbird/Thunderbird.log") #Might take 2-3 minutes in HPC cloud. In desktop out of memory
-    elif dataset=="bgl":
-        loader = load.BGLLoader(filename=f"{full_data}/bgl/BGL.log")
-    elif(dataset=="hadoop"):    
-        loader = load.HadoopLoader(filename=f"{full_data}/hadoop/",
-                                        filename_pattern  ="*.log",
-                                        labels_file_name=f"{full_data}/hadoop/abnormal_label_accurate.txt")
+    if dataset == "hdfs":
+        loader = HDFSLoader(filename=f"{full_data}/hdfs/HDFS.log",
+                            labels_file_name=f"{full_data}/hdfs/anomaly_label.csv")
+    elif dataset == "tb":
+        loader = ThuSpiLibLoader(filename=f"{full_data}/thunderbird/Thunderbird.log") #Might take 2-3 minutes in HPC cloud. In desktop out of memory
+    elif dataset == "bgl":
+        loader = BGLLoader(filename=f"{full_data}/bgl/BGL.log")
+    elif dataset == "hadoop":
+        loader = HadoopLoader(filename=f"{full_data}/hadoop/",
+                              filename_pattern  ="*.log",
+                              labels_file_name=f"{full_data}/hadoop/abnormal_label_accurate.txt")
         
     return loader
 
@@ -52,11 +52,11 @@ def create_correct_loader(dataset):
     dataset_info["without_masking"][dataset_name]['row_count'] = row_count
     dataset_info["with_masking"][dataset_name]['row_count'] = row_count
     
-    enhancer = er.EventLogEnhancer(df)
+    enhancer = EventLogEnhancer(df)
 
     # Running without drain_masking
     for _ in range(10):
-        enhancer_copy = er.EventLogEnhancer(df)  # Use a fresh copy for each iteration
+        enhancer_copy = EventLogEnhancer(df)  # Use a fresh copy for each iteration
         time_start = time.time()
         df_temp = enhancer_copy.normalize()
         df_temp = enhancer_copy.parse_drain(reparse=True)
@@ -65,7 +65,7 @@ def create_correct_loader(dataset):
 
     # Running with drain_masking
     for _ in range(10):
-        enhancer_copy = er.EventLogEnhancer(df)  # Use a fresh copy for each iteration
+        enhancer_copy = EventLogEnhancer(df)  # Use a fresh copy for each iteration
         time_start = time.time()
         df_temp = enhancer_copy.parse_drain(drain_masking=True, reparse=True)
         elapsed_time = time.time() - time_start
diff --git a/demo/saner_2024_paper/t3_parser_comparison.py b/demo/saner_2024_paper/t3_parser_comparison.py
index 8ff99ae..9e2a481 100644
--- a/demo/saner_2024_paper/t3_parser_comparison.py
+++ b/demo/saner_2024_paper/t3_parser_comparison.py
@@ -2,19 +2,19 @@
 # LogParsers loading routine has been slightly modified. Modifications are explained in the separate file
  
 import sys
-import time
 sys.path.append('..')
-import polars as pl
 import copy
-import loglead.loaders.base as load
-import loglead.enhancer as er
-from parsers.bert.bertembedding import BertEmbeddings
-full_data = "/home/ubuntu/Datasets"
+import time
+import statistics
 
+from loglead.enhancers import EventLogEnhancer
+from loglead.loaders import HDFSLoader
 
+full_data = "/home/ubuntu/Datasets"
 
-loader = load.HDFSLoader(filename=f"{full_data}/hdfs/HDFS.log", 
-                                            labels_file_name=f"{full_data}/hdfs/anomaly_label.csv")
+
+loader = HDFSLoader(filename=f"{full_data}/hdfs/HDFS.log",
+                    labels_file_name=f"{full_data}/hdfs/anomaly_label.csv")
 
 
 loader.execute()
@@ -23,19 +23,17 @@
 df100k = copy.deepcopy(loader).reduce_dataframes(0.01)
 df200k = copy.deepcopy(loader).reduce_dataframes(0.02)
 
-datas =  [df50k, df100k, df200k]
-parsers = ["drain", "spell", "lenma", "neural"]
-parsers = ["neural"]
+datas = [df50k, df100k, df200k]
+# parsers = ["drain", "spell", "lenma", "neural"]
+# parsers = ["neural"]
 parsers = ["drain", "spell", "lenma"]
 
-import time
-import statistics
 for data in datas:
     df = data
     for parser in parsers:
         elapsed_times = []
         for _ in range(10):
-            enricher = er.EventLogEnhancer(df)
+            enricher = EventLogEnhancer(df)
             start_time = time.time()
             df = enricher.normalize()
             if parser == "drain":
@@ -49,7 +47,6 @@
             elif parser == "neural":
                 df_parsers = enricher.create_neural_emb()
             
-
             end_time = time.time()
             elapsed_time = end_time - start_time
             elapsed_times.append(elapsed_time)
diff --git a/demo/saner_2024_paper/t4_ano_detect_comparison.py b/demo/saner_2024_paper/t4_ano_detect_comparison.py
index 12fe909..c775e60 100644
--- a/demo/saner_2024_paper/t4_ano_detect_comparison.py
+++ b/demo/saner_2024_paper/t4_ano_detect_comparison.py
@@ -1,29 +1,33 @@
-
 import sys
-import time
 sys.path.append('..')
+
 import polars as pl
-import loglead.loaders.base as load, loglead.enhancer as er, loglead.anomaly_detection as ad
+
+from loglead import AnomalyDetector
+from loglead.enhancers import EventLogEnhancer, SequenceEnhancer
+from loglead.loaders import HDFSLoader
+
+
 full_data = "/home/ubuntu/Datasets"
-private_data ="../private_data"
+private_data = "../private_data"
 
-loader = load.HDFSLoader(filename=f"{full_data}/hdfs/HDFS.log", 
-                                            labels_file_name=f"{full_data}/hdfs/anomaly_label.csv")
+loader = HDFSLoader(filename=f"{full_data}/hdfs/HDFS.log",
+                    labels_file_name=f"{full_data}/hdfs/anomaly_label.csv")
 loader.execute()
 
 #Reduce as needed.
 #df = loader.reduce_dataframes(0.001)
 #df = loader.reduce_dataframes(0.1)
 df = loader.df
-df_seq =  loader.df_sequences
+df_seq = loader.df_sequences
 
-enchancer = er.EventLogEnhancer(df)
-df = enchancer.normalize()
-df = enchancer.words(column="e_message_normalized")
-df = enchancer.parse_drain()
-df = enchancer.parse_spell()
-df = enchancer.parse_lenma()
-df = enchancer.create_neural_emb()
+enhancer = EventLogEnhancer(df)
+df = enhancer.normalize()
+df = enhancer.words(column="e_message_normalized")
+df = enhancer.parse_drain()
+df = enhancer.parse_spell()
+df = enhancer.parse_lenma()
+df = enhancer.create_neural_emb()
 
 #Perform an intermediate save here. 
 #df.write_parquet(f"{private_data}/P4_Parsed_hdfs_events_01.parquet")
@@ -35,7 +39,7 @@
 df_seq = pl.read_parquet(f"{private_data}/P4_Parsed_hdfs_seqs_01.parquet")
 
 
-seq_enhancer = er.SequenceEnhancer(df = df, df_seq = df_seq)
+seq_enhancer = SequenceEnhancer(df=df, df_seq=df_seq)
 seq_enhancer.tokens("e_words")
 seq_enhancer.events("e_event_id")
 seq_enhancer.events("e_event_lenma_id")
@@ -48,14 +52,14 @@
 from sklearn.exceptions import ConvergenceWarning
 warnings.filterwarnings("ignore", category=ConvergenceWarning)
 #Disable unsupervised
-disable = ["train_IsolationForest", "train_LOF","train_KMeans", "train_OneClassSVM", "train_RarityModel"]
+disable = ["train_IsolationForest", "train_LOF", "train_KMeans", "train_OneClassSVM", "train_RarityModel"]
 #Need a loop to run these multiple times
-sad = ad.AnomalyDetection(store_scores=True, print_scores=False)
+sad = AnomalyDetector(store_scores=True, print_scores=False)
 
 for i in range(10):
-    print (f"{i}", end ="")
+    print(f"{i}", end="")
     sad.item_list_col = "e_words"
-    sad.test_train_split (seq_enhancer.df_seq, test_frac=0.95)
+    sad.test_train_split(seq_enhancer.df_seq, test_frac=0.95)
     sad.evaluate_all_ads(disabled_methods=disable)
     print(".", end="")
     
@@ -64,16 +68,16 @@
     sad.evaluate_all_ads(disabled_methods=disable)
     print(".", end="")
     
-    sad.item_list_col ="e_event_lenma_id"
+    sad.item_list_col = "e_event_lenma_id"
     sad.prepare_train_test_data()
     sad.evaluate_all_ads(disabled_methods=disable)
     print(".", end="")
     
-    sad.item_list_col ="e_event_spell_id"
+    sad.item_list_col = "e_event_spell_id"
     sad.prepare_train_test_data()
     sad.evaluate_all_ads(disabled_methods=disable)
     print(".")
-    sad.item_list_col =None
+    sad.item_list_col = None
     
     sad.emb_list_col = "e_bert_emb"
     sad.prepare_train_test_data()
@@ -82,4 +86,3 @@
 
 #-----------------------------------------------
 print(sad.storage.calculate_average_scores(score_type="f1").to_csv())
-
diff --git a/demo/unsupervised_models.py b/demo/unsupervised_models.py
index c127e98..0e32a5c 100644
--- a/demo/unsupervised_models.py
+++ b/demo/unsupervised_models.py
@@ -1,27 +1,28 @@
-#
-#Separate demo files
 import sys
 sys.path.append('..')
+import time
+
+import polars as pl
+
+from loglead.loaders import BGLLoader
+from loglead.loaders import ThuSpiLibLoader
+from loglead.loaders import HadoopLoader
+from loglead.loaders import HDFSLoader
+from loglead.enhancers import EventLogEnhancer, SequenceEnhancer
+from loglead import AnomalyDetector
+
 # Suppress ConvergenceWarning
-import warnings
 from sklearn.exceptions import ConvergenceWarning
+import warnings
 warnings.filterwarnings("ignore", category=ConvergenceWarning)
 
-import loglead.loaders.supercomputers as load_sc
-import loglead.loaders.hadoop as load_hadoop
-import loglead.loaders.hdfs as load_hdfs
-
-import loglead.enhancer as er, loglead.anomaly_detection as ad
-import polars as pl
-import time 
-
-#Adjust full data source
+# Adjust full data source
 full_data = "/home/ubuntu/Datasets"
 
-#List the representations (column names) for anomaly detection
-items = ["e_words", "e_trigrams","e_event_drain_id"] #"e_alphanumerics"
+# List the representations (column names) for anomaly detection
+items = ["e_words", "e_trigrams", "e_event_drain_id"]  # "e_alphanumerics"
 
-#List the models as well as possible parameters
+# List the models as well as possible parameters
 models_dict = {
     "IsolationForest": {},
     "KMeans": {},
@@ -30,19 +31,19 @@
 }
 
 # Notes:
-### F1 scores for these models don't tell much as the threshold adjustment is so crucial.
-### Each dataset has their own section where, e.g., you can determine how much data to use. 
-### While the code is mostly the same, event and sequence based datasets have some differences 
-### Whether the model uses completely unfiltered data (i.e. anomalies in training) can be adjusted with parameter "filter_anos"
-### Here normalization is simply turning to lowercase, all numbers to 0s and multiple subsequent 0s to single 0
+# F1 scores for these models don't tell much as the threshold adjustment is so crucial.
+# Each dataset has their own section where, e.g., you can determine how much data to use.
+# While the code is mostly the same, event and sequence based datasets have some differences
+# Whether the model uses completely unfiltered data (i.e. anomalies in training) can be adjusted with parameter "filter_anos"
+# Here normalization is simply turning to lowercase, all numbers to 0s and multiple subsequent 0s to single 0
 
 print("---------- Hadoop ----------")
 frac_data = 1
 test_frac = 0.5
 stime = time.time()
-loader = load_hadoop.HadoopLoader(filename=f"{full_data}/hadoop/",
-                                            filename_pattern  ="*.log",
-                                            labels_file_name=f"{full_data}/hadoop/abnormal_label_accurate.txt")
+loader = HadoopLoader(filename=f"{full_data}/hadoop/",
+                      filename_pattern="*.log",
+                      labels_file_name=f"{full_data}/hadoop/abnormal_label_accurate.txt")
 df = loader.execute()
 df = loader.reduce_dataframes(frac=frac_data)
 
@@ -50,10 +51,10 @@
 print("time loaded", time.time()-stime)
 df = df.filter(pl.col("m_message").is_not_null())
 
-enhancer =  er.EventLogEnhancer(df)
+enhancer = EventLogEnhancer(df)
 df = enhancer.length()
 
-regexs = [('0','\d'),('0','0+')]
+regexs = [('0', '\d'), ('0', '0+')]
 df = enhancer.normalize(regexs, to_lower=True)
 print("time normalized", time.time()-stime)
 stime = time.time()
@@ -67,13 +68,13 @@
 print("time parse", time.time()-stime)
 stime = time.time()
 
-seq_enhancer = er.SequenceEnhancer(df = df, df_seq = df_seq)
-print("ano", len(seq_enhancer.df_seq.filter(seq_enhancer.df_seq["normal"]==False)))
-print("normal", len(seq_enhancer.df_seq.filter(seq_enhancer.df_seq["normal"]==True)))
-seq_enhancer.seq_len() #OOVD uses data from the df for faster calculations
+seq_enhancer = SequenceEnhancer(df=df, df_seq=df_seq)
+print("ano", len(seq_enhancer.df_seq.filter(seq_enhancer.df_seq["normal"] is False)))
+print("normal", len(seq_enhancer.df_seq.filter(seq_enhancer.df_seq["normal"] is True)))
+seq_enhancer.seq_len()  # OOVD uses data from the df for faster calculations
 seq_enhancer.start_time()
 
-sad = ad.AnomalyDetection()
+sad = AnomalyDetector()
 for item in items:
     print("-----", item, "-----")
     if item != "e_event_drain_id":
@@ -84,7 +85,7 @@
 
     stime = time.time()
     seq_enhancer.sort_start_time()
-    sad.test_train_split (seq_enhancer.df_seq, test_frac=test_frac)
+    sad.test_train_split(seq_enhancer.df_seq, test_frac=test_frac)
     print("time split and prepare:", time.time()-stime)
 
     sad.evaluate_with_params(models_dict)
@@ -94,17 +95,17 @@
 frac_data = 0.01
 test_frac = 0.95
 stime = time.time()
-loader = load_sc.BGLLoader(filename=f"{full_data}/bgl/BGL.log")
+loader = BGLLoader(filename=f"{full_data}/bgl/BGL.log")
 df = loader.execute()
-print("ano", len(df.filter(df["normal"]==False)))
-print("normal", len(df.filter(df["normal"]==True)))
+print("ano", len(df.filter(df["normal"] is False)))
+print("normal", len(df.filter(df["normal"] is True)))
 df = loader.reduce_dataframes(frac=frac_data)
 df = df.filter(pl.col("m_message").is_not_null())
 print("time loaded", time.time()-stime)
 
-enhancer =  er.EventLogEnhancer(df)
+enhancer = EventLogEnhancer(df)
 stime = time.time()
-regexs = [('0','\d'),('0','0+')]
+regexs = [('0', '\d'), ('0', '0+')]
 df = enhancer.normalize(regexs, to_lower=True)
 print("time normalized", time.time()-stime)
 stime = time.time()
@@ -120,12 +121,12 @@
 
 df = enhancer.length("e_message_normalized")
 
-sad = ad.AnomalyDetection() 
+sad = AnomalyDetector()
 for item in items:
     print("-----", item, "-----")
     sad.item_list_col = item
     stime = time.time()
-    sad.test_train_split (df, test_frac=test_frac)
+    sad.test_train_split(df, test_frac=test_frac)
     print("time split and prepare:", time.time()-stime)
     sad.evaluate_with_params(models_dict)
 
@@ -134,19 +135,19 @@
 frac_data = 0.01
 test_frac = 0.95
 stime = time.time()
-loader = load_hdfs.HDFSLoader(filename=f"{full_data}/hdfs/HDFS.log", 
-                                    labels_file_name=f"{full_data}/hdfs/anomaly_label.csv")
+loader = HDFSLoader(filename=f"{full_data}/hdfs/HDFS.log",
+                    labels_file_name=f"{full_data}/hdfs/anomaly_label.csv")
 df = loader.execute()
 df = loader.reduce_dataframes(frac=frac_data)
 df_seq = loader.df_seq       
 print("time loaded", time.time()-stime)
 
 df = df.filter(pl.col("m_message").is_not_null())
-enhancer =  er.EventLogEnhancer(df)
+enhancer = EventLogEnhancer(df)
 df = enhancer.length()
 
 
-regexs = [('0','\d'),('0','0+')]
+regexs = [('0', '\d'), ('0', '0+')]
 df = enhancer.normalize(regexs, to_lower=True)
 print("time normalized", time.time()-stime)
 stime = time.time()
@@ -160,12 +161,12 @@
 print("time parse", time.time()-stime)
 stime = time.time()
 
-seq_enhancer = er.SequenceEnhancer(df = df, df_seq = df_seq)
-print("ano", len(seq_enhancer.df_seq.filter(seq_enhancer.df_seq["normal"]==False)))
-print("normal", len(seq_enhancer.df_seq.filter(seq_enhancer.df_seq["normal"]==True)))
+seq_enhancer = SequenceEnhancer(df=df, df_seq=df_seq)
+print("ano", len(seq_enhancer.df_seq.filter(seq_enhancer.df_seq["normal"] is False)))
+print("normal", len(seq_enhancer.df_seq.filter(seq_enhancer.df_seq["normal"] is True)))
 seq_enhancer.seq_len()
 
-sad = ad.AnomalyDetection()
+sad = AnomalyDetector()
 for item in items:
     print("-----", item, "-----")
     if item != "e_event_drain_id":
@@ -175,7 +176,7 @@
     sad.item_list_col = item
 
     stime = time.time()
-    sad.test_train_split (seq_enhancer.df_seq, test_frac=test_frac)
+    sad.test_train_split(seq_enhancer.df_seq, test_frac=test_frac)
     print("time split and prepare:", time.time()-stime)
     sad.evaluate_with_params(models_dict)
 
@@ -184,19 +185,19 @@
 frac_data = 0.001
 test_frac = 0.95
 stime = time.time()
-loader = load_sc.ThuSpiLibLoader(filename=f"{full_data}/thunderbird/Thunderbird.log", split_component=False)
+loader = ThuSpiLibLoader(filename=f"{full_data}/thunderbird/Thunderbird.log", split_component=False)
 df = loader.execute()
-print("ano", len(df.filter(df["normal"]==False)))
-print("normal", len(df.filter(df["normal"]==True)))
+print("ano", len(df.filter(df["normal"] is False)))
+print("normal", len(df.filter(df["normal"] is True)))
 df = loader.reduce_dataframes(frac=frac_data)
 df = df.filter(pl.col("m_message").is_not_null())
-enhancer =  er.EventLogEnhancer(df)
+enhancer = EventLogEnhancer(df)
 print("time loaded", time.time()-stime)
 
 
-enhancer =  er.EventLogEnhancer(df)
+enhancer = EventLogEnhancer(df)
 stime = time.time()
-regexs = [('0','\d'),('0','0+')]
+regexs = [('0', '\d'), ('0', '0+')]
 df = enhancer.normalize(regexs, to_lower=True)
 print("time normalized", time.time()-stime)
 stime = time.time()
@@ -212,13 +213,13 @@
 
 df = enhancer.length("e_message_normalized")
 
-sad = ad.AnomalyDetection() 
+sad = AnomalyDetector()
 for item in items:
     print("-----", item, "-----")
     sad.item_list_col = item
     
     stime = time.time()
-    sad.test_train_split (df, test_frac=test_frac)
+    sad.test_train_split(df, test_frac=test_frac)
     print("time split and prepare:", time.time()-stime)
     
-    sad.evaluate_with_params(models_dict)
\ No newline at end of file
+    sad.evaluate_with_params(models_dict)
diff --git a/loglead/OOV_detector.py b/loglead/OOV_detector.py
index 2b78f87..d9b21fb 100644
--- a/loglead/OOV_detector.py
+++ b/loglead/OOV_detector.py
@@ -1,24 +1,23 @@
-
-
 import matplotlib.pyplot as plt
-import matplotlib as mpl
-import math
 import polars as pl
 import numpy as np
 
+__all__ = ['OOV_detector']
+
+
 class OOV_detector:
-    def __init__(self, len_col, test_df, threshold = 1):
+    def __init__(self, len_col, test_df, threshold=1):
         self.len_col = len_col
         self.test_df = test_df
         self.scores = 0
         self.threshold = threshold
             
-    def fit(self, X_train = None, labels = None):
-        #The "training set" to compare against comes inherently from test data's sparse matrix
+    def fit(self, X_train=None, labels=None):
+        # The "training set" to compare against comes inherently from test data's sparse matrix
         return
     
     def predict(self, X_test):
-        #Give array of 0s if the needed length column is lacking in the df        
+        # Give array of 0s if the needed length column is lacking in the df
         if self.len_col not in self.test_df.columns:
             print("Column not found for OOVD")
             return np.zeros(self.test_df.select(pl.count()).item())
@@ -32,7 +31,7 @@ def predict(self, X_test):
     
     def custom_plot(self, labels, x_axis_scale=1.0):
         # Double the font size
-        #mpl.rcParams.update({'font.size': mpl.rcParams['font.size']*1.5})
+        # mpl.rcParams.update({'font.size': mpl.rcParams['font.size']*1.5})
         
         labels_bool = np.array(labels).astype(bool)
         scores_norm = self.scores[~labels_bool]
@@ -51,4 +50,3 @@ def custom_plot(self, labels, x_axis_scale=1.0):
 
         plt.tight_layout()
         plt.show()
-
diff --git a/loglead/RarityModel.py b/loglead/RarityModel.py
index b05fd92..f7d61ca 100644
--- a/loglead/RarityModel.py
+++ b/loglead/RarityModel.py
@@ -1,10 +1,10 @@
-import matplotlib.pyplot as plt
-import matplotlib as mpl
 import math
-import time
-import polars as pl
+
+import matplotlib.pyplot as plt
 import numpy as np
 
+__all__ = ['RarityModel']
+
 
 class RarityModel:
     def __init__(self, threshold = 10, common_threshold = 0.01):
@@ -14,8 +14,7 @@ def __init__(self, threshold = 10, common_threshold = 0.01):
         self.is_norm = None
         self.common_threshold = common_threshold
         
-    
-    def fit(self, X_train, labels=None):  
+    def fit(self, X_train, labels=None):
         def rarity_score(freq, total_ngrams):
             normalized_freq = freq / total_ngrams
             if normalized_freq > self.common_threshold:
@@ -31,18 +30,17 @@ def rarity_score(freq, total_ngrams):
         train_counts = np.array(X_train.sum(axis=0))[0]
         self.score_vector = np.array([rarity_score(count, total_ngrams) for count in train_counts])
         
-    
     def predict(self, X_test):
         X_test_csr = X_test.tocsr()
-        #Getting the count of non-zero elements along axis 1 (columns) for each instance
+        # Getting the count of non-zero elements along axis 1 (columns) for each instance
         non_zero_counts = np.array(X_test_csr.getnnz(axis=1), dtype=np.float64)  #Convert to float64 here
         non_zero_counts[non_zero_counts == 0] = 1  #ensuring no divisions by 0
         self.scores = X_test_csr.dot(self.score_vector)
-        #Ensuring self.scores is a float array
+        # Ensuring self.scores is a float array
         self.scores = self.scores.astype(np.float64)
-        #Dividing the scores by the count of non-zero elements
+        # Dividing the scores by the count of non-zero elements
         self.scores /= non_zero_counts
-        #Comparing the scores to the threshold
+        # Comparing the scores to the threshold
         self.is_ano = (self.scores > self.threshold).astype(int)
         return self.is_ano
     
diff --git a/loglead/__init__.py b/loglead/__init__.py
index e69de29..d3c139e 100644
--- a/loglead/__init__.py
+++ b/loglead/__init__.py
@@ -0,0 +1,6 @@
+from .anomaly_detection import AnomalyDetector
+from .OOV_detector import OOV_detector
+from .RarityModel import RarityModel
+from .next_event_prediction import NextEventPredictionNgram
+
+__all__ = ['AnomalyDetector', 'OOV_detector', 'RarityModel', 'NextEventPredictionNgram']
\ No newline at end of file
diff --git a/loglead/anomaly_detection.py b/loglead/anomaly_detection.py
index b39750f..d97489b 100644
--- a/loglead/anomaly_detection.py
+++ b/loglead/anomaly_detection.py
@@ -1,11 +1,15 @@
+import time
+
 import polars as pl
 import numpy as np
-from collections import Counter
+import pandas as pd
+import matplotlib.pyplot as plt
 #Faster sklearn enabled. See https://intel.github.io/scikit-learn-intelex/latest/
 # Causes problems in RandomForrest. We have to use older version due to tensorflow numpy combatibilities
 # from sklearnex import patch_sklearn
 #patch_sklearn()
-from sklearn.feature_extraction.text import CountVectorizer
+from scipy.sparse import hstack
+from xgboost import XGBClassifier
 from sklearn.linear_model import LogisticRegression
 from sklearn.tree import DecisionTreeClassifier
 from sklearn.svm import LinearSVC
@@ -13,30 +17,21 @@
 from sklearn.neighbors import LocalOutlierFactor
 from sklearn.cluster import KMeans
 from sklearn.ensemble import RandomForestClassifier
-from xgboost import XGBClassifier
 from sklearn.svm import OneClassSVM
 from sklearn.metrics import f1_score
 from sklearn.metrics import confusion_matrix
-import csv
-from io import StringIO
-import pandas as pd
-from loglead.RarityModel import RarityModel
-from loglead.OOV_detector import OOV_detector
-
-
 from sklearn.metrics import accuracy_score
-from scipy.sparse import hstack
-import scipy.sparse
-import math
-import time
-
-from scipy.sparse import csr_matrix
-import matplotlib.pyplot as plt
 from sklearn.metrics import roc_curve, auc
-from sklearn.feature_extraction.text import TfidfVectorizer
 from sklearn.metrics import roc_auc_score
+from sklearn.feature_extraction.text import CountVectorizer
+
+from .RarityModel import RarityModel
+from .OOV_detector import OOV_detector
+
+__all__ = ['AnomalyDetector']
+
 
-class AnomalyDetection:
+class AnomalyDetector:
     def __init__(self, item_list_col=None, numeric_cols=None, emb_list_col=None, label_col="anomaly", 
                  store_scores=False, print_scores=True, auc_roc=False):
         self.item_list_col = item_list_col
@@ -44,13 +39,12 @@ def __init__(self, item_list_col=None, numeric_cols=None, emb_list_col=None, lab
         self.label_col = label_col
         self.emb_list_col = emb_list_col
         self.store_scores = store_scores
-        self.storage = ModelResultsStorage()
+        self.storage = _ModelResultsStorage()
         self.print_scores=print_scores
         self.train_vocabulary = None
         self.auc_roc = auc_roc
 
-        
-    def test_train_split(self, df, test_frac=0.9, shuffle=True,vec_name="CountVectorizer"):
+    def test_train_split(self, df, test_frac=0.9, shuffle=True, vectorizer_class=CountVectorizer):
         # Shuffle the DataFrame
         if shuffle:
             df = df.sample(fraction = 1.0, shuffle=True)
@@ -65,18 +59,18 @@ def test_train_split(self, df, test_frac=0.9, shuffle=True,vec_name="CountVector
         # Split the DataFrame using head and tail
         self.train_df = df.head(-test_size) #Returns all rows expect last abs(-test_size)
         self.test_df = df.tail(test_size) #Returns the last test_size rows
-        self.prepare_train_test_data(vec_name=vec_name)
+        self.prepare_train_test_data(vectorizer_class=vectorizer_class)
         
-    def prepare_train_test_data(self, vec_name="CountVectorizer"):
+    def prepare_train_test_data(self, vectorizer_class=CountVectorizer):
         #Prepare all data for running
-        self.X_train, self.labels_train = self._prepare_data(True, self.train_df, vec_name)
-        self.X_test, self.labels_test = self._prepare_data(False, self.test_df,vec_name)
+        self.X_train, self.labels_train = self._prepare_data(True, self.train_df, vectorizer_class)
+        self.X_test, self.labels_test = self._prepare_data(False, self.test_df, vectorizer_class)
         #No anomalies dataset is used for some unsupervised algos. 
-        self.X_train_no_anos, _ = self._prepare_data(True, self.train_df.filter(pl.col(self.label_col).not_()), vec_name)
-        self.X_test_no_anos, self.labels_test_no_anos = self._prepare_data(False, self.test_df, vec_name)
+        self.X_train_no_anos, _ = self._prepare_data(True, self.train_df.filter(pl.col(self.label_col).not_()),
+                                                     vectorizer_class)
+        self.X_test_no_anos, self.labels_test_no_anos = self._prepare_data(False, self.test_df, vectorizer_class)
      
-        
-    def _prepare_data(self, train, df_seq, vec_name):
+    def _prepare_data(self, train, df_seq, vectorizer_class):
         X = None
         labels = df_seq.select(pl.col(self.label_col)).to_series().to_list()
 
@@ -85,7 +79,6 @@ def _prepare_data(self, train, df_seq, vec_name):
             # Extract the column
             column_data = df_seq.select(pl.col(self.item_list_col))             
             events = column_data.to_series().to_list()
-            vectorizer_class = globals()[vec_name]
             # We are training
             if train:
                 # Check the datatype  
@@ -117,8 +110,6 @@ def _prepare_data(self, train, df_seq, vec_name):
 
         return X, labels    
         
-
-         
     def train_model(self, model, filter_anos=False):
         X_train_to_use = self.X_train_no_anos if filter_anos else self.X_train
         #Store the current the model and whether it uses ano data or no
@@ -165,8 +156,6 @@ def predict(self, custom_plot=False):
                 predictions_proba = self.model.predict_proba(X_test_to_use)[:, 1]
             df_seq = self.test_df.with_columns(pl.Series(name="pred_ano_proba", values=predictions_proba.tolist()))      
 
-
-
         if self.print_scores:
             self._print_evaluation_scores(self.labels_test, predictions,predictions_proba, self.model)
         if custom_plot:
@@ -220,16 +209,17 @@ def train_OOVDetector(self, len_col=None, filter_anos=True, threshold=1):
                 len_col = self.item_list_col+"_len"
         self.train_model(OOV_detector(len_col, self.test_df, threshold), filter_anos=filter_anos)
         
-
-    def evaluate_all_ads(self, disabled_methods=[]):
-        for method_name in sorted(dir(self)): 
-            if (method_name.startswith("train_") 
-                and not method_name.startswith("train_model") 
-                and method_name not in disabled_methods):
+    def evaluate_all_ads(self, disabled_methods=None):
+        if disabled_methods is None:
+            disabled_methods = []
+        for method_name in sorted(dir(self)):
+            if (method_name.startswith("train_")
+                    and not method_name.startswith("train_model")
+                    and method_name not in disabled_methods):
                 method = getattr(self, method_name)
                 if callable(method):
                     if not self.print_scores:
-                        print (f"Running {method_name}")
+                        print(f"Running {method_name}")
                     time_start = time.time()
                     method()
                     self.predict()
@@ -245,8 +235,7 @@ def evaluate_with_params(self, models_dict):
             method(**params)
             self.predict()
 
-
-    def _print_evaluation_scores(self, y_test, y_pred,y_pred_proba, model, f_importance = False, auc_roc = True):
+    def _print_evaluation_scores(self, y_test, y_pred, y_pred_proba, model, f_importance=False, auc_roc=True):
         print(f"Results from model: {type(model).__name__}")
         # Evaluate the model's performance
         accuracy = accuracy_score(y_test, y_pred)
@@ -294,16 +283,38 @@ def _print_evaluation_scores(self, y_test, y_pred,y_pred_proba, model, f_importa
             X_test_to_use = self.X_test_no_anos if self.filter_anos else self.X_test
             if isinstance(self.model, IsolationForest):
                 y_pred = 1 - model.score_samples(X_test_to_use) #lower = anomalous
-                print(f"AUCROC: {auc_roc_analysis(y_test, y_pred, titlestr):.4f}")
+                print(f"AUCROC: {self._auc_roc_analysis(y_test, y_pred, titlestr):.4f}")
             if isinstance(self.model, KMeans):
                 y_pred = np.min(model.transform(X_test_to_use), axis=1) #Shortest distance from the cluster to be used as ano score
-                print(f"AUCROC: {auc_roc_analysis(y_test, y_pred, titlestr):.4f}")
+                print(f"AUCROC: {self._auc_roc_analysis(y_test, y_pred, titlestr):.4f}")
             if isinstance(self.model, (RarityModel, OOV_detector)):
-                print(f"AUCROC: {auc_roc_analysis(y_test,  model.scores, titlestr):.4f}")
-
-
-
-class ModelResultsStorage:
+                print(f"AUCROC: {self._auc_roc_analysis(y_test, model.scores, titlestr):.4f}")
+
+    @staticmethod
+    def _auc_roc_analysis(labels, preds, titlestr ="ROC", plot=False):
+        # Compute the ROC curve
+        fpr, tpr, thresholds = roc_curve(labels, preds)
+        # Compute the AUC from the points of the ROC curve
+        roc_auc = auc(fpr, tpr)
+
+        if plot:
+            # Plot the ROC curve
+            plt.figure()
+            lw = 2
+            plt.plot(fpr, tpr, color='darkorange', lw=lw, label='ROC curve (area = %0.2f)' % roc_auc)
+            plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')
+            plt.xlim([0.0, 1.0])
+            plt.ylim([0.0, 1.05])
+            plt.xlabel('False Positive Rate')
+            plt.ylabel('True Positive Rate')
+            plt.title(titlestr)
+            plt.legend(loc="lower right")
+            plt.show()
+
+        return roc_auc
+
+
+class _ModelResultsStorage:
     def __init__(self):
         self.test_results = []
 
@@ -405,9 +416,6 @@ def calculate_average_scores(self, score_type='accuracy', metric='mean', mark_mo
         df_pivot = df_pivot.map(lambda x: f"{x:.3f}")
         return df_pivot
         
-        
-        return df_pivot
-
     def print_confusion_matrices(self, model_filter=None, signature_filter=None):
         for result in self.test_results:
             #model_name = type(result['model']).__name__
@@ -428,7 +436,6 @@ def print_confusion_matrices(self, model_filter=None, signature_filter=None):
             print("Confusion Matrix:")
             print(cm)
             
-
     def print_scores(self, model_filter=None, signature_filter=None, score_type='all'):
         # Check for valid score_type
         valid_score_types = ['accuracy', 'f1', 'all']
@@ -457,37 +464,3 @@ def print_scores(self, model_filter=None, signature_filter=None, score_type='all
                 print(f"F1 Score: {f1:.4f}")
 
 
-
-def auc_roc_analysis(labels, preds, titlestr = "ROC", plot=False):
-    # Compute the ROC curve
-    fpr, tpr, thresholds = roc_curve(labels, preds)
-    # Compute the AUC from the points of the ROC curve
-    roc_auc = auc(fpr, tpr)
-
-    if plot:
-        # Plot the ROC curve
-        plt.figure()
-        lw = 2
-        plt.plot(fpr, tpr, color='darkorange', lw=lw, label='ROC curve (area = %0.2f)' % roc_auc)
-        plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')
-        plt.xlim([0.0, 1.0])
-        plt.ylim([0.0, 1.05])
-        plt.xlabel('False Positive Rate')
-        plt.ylabel('True Positive Rate')
-        plt.title(titlestr)
-        plt.legend(loc="lower right")
-        plt.show()
-
-    return roc_auc
-
-    
-def test_train_split(df, test_frac):
-    # Shuffle the DataFrame
-    df = df.sample(fraction = 1.0, shuffle=True)
-    # Split ratio
-    test_size = int(test_frac * df.shape[0])
-
-    # Split the DataFrame using head and tail
-    test_df = df.head(test_size)
-    train_df = df.tail(-test_size)
-    return train_df, test_df
diff --git a/loglead/enhancers/__init__.py b/loglead/enhancers/__init__.py
new file mode 100644
index 0000000..e4153dd
--- /dev/null
+++ b/loglead/enhancers/__init__.py
@@ -0,0 +1,4 @@
+from .eventlog import EventLogEnhancer
+from .sequence import SequenceEnhancer
+
+__all__ = ['EventLogEnhancer', 'SequenceEnhancer']
\ No newline at end of file
diff --git a/loglead/enhancer.py b/loglead/enhancers/eventlog.py
similarity index 69%
rename from loglead/enhancer.py
rename to loglead/enhancers/eventlog.py
index 6fa8370..f3419c5 100644
--- a/loglead/enhancer.py
+++ b/loglead/enhancers/eventlog.py
@@ -1,11 +1,8 @@
-import polars as pl
-
-
 import hashlib
-#Lazy import inside the method. 
+
+import polars as pl
+#Lazy import inside the method.
 #from .bertembedding import BertEmbeddings
-import os
-import loglead.next_event_prediction as nep
 
 # Drain.ini default regexes
 # No lookahead or lookbedinde so reimplemented with capture groups. Still problem with overlaps See
@@ -27,6 +24,8 @@
     ("${cmd}<CMD>", r"(?P<cmd>executed cmd )(\".+?\")")
 ]
 
+__all__ = ['EventLogEnhancer']
+
 
 class EventLogEnhancer:
     def __init__(self, df):
@@ -82,7 +81,8 @@ def trigrams(self, column="m_message"):
             )
         return self.df
 
-    def _create_cngram(self, message, ngram=3):
+    @staticmethod
+    def _create_cngram(message, ngram=3):
         if ngram <= 0:
             return []
         return [message[i:i + ngram] for i in range(len(message) - ngram + 1)]
@@ -91,7 +91,6 @@ def _create_cngram(self, message, ngram=3):
     def parse_drain(self, field = "e_message_normalized", drain_masking=False, reparse=False, templates=False):
         self._handle_prerequisites([field])
         if reparse or "e_event_drain_id" not in self.df.columns:
-            import drain3 as dr
             # Drain returns dict
             # {'change_type': 'none',
             # 'cluster_id': 1,
@@ -102,9 +101,6 @@ def parse_drain(self, field = "e_message_normalized", drain_masking=False, repar
 
             # We might have multiline log message, i.e. log_message + stack trace.
             # Use only first line of log message for parsing
-            current_script_path = os.path.abspath(__file__)
-            current_script_directory = os.path.dirname(current_script_path)
-            drain3_ini_location =  os.path.join(current_script_directory, '../parsers/drain3/')
             return_dtype = pl.Struct([
                 pl.Field("change_type", pl.Utf8),
                 pl.Field("cluster_id", pl.Int64),
@@ -113,8 +109,7 @@ def parse_drain(self, field = "e_message_normalized", drain_masking=False, repar
                 pl.Field("cluster_count", pl.Int64)
             ])
             if drain_masking:
-                dr.template_miner.config_filename = os.path.join(drain3_ini_location,'drain3.ini') #TODO fix the path relative
-                tm = dr.TemplateMiner()
+                from loglead.parsers import DrainTemplateMiner as tm
                 self.df = self.df.with_columns(
                     message_trimmed=pl.col("m_message").str.split("\n").list.first()
                 )
@@ -123,8 +118,7 @@ def parse_drain(self, field = "e_message_normalized", drain_masking=False, repar
             else:
                 #if "e_message_normalized" not in self.df.columns:
                 #    self.normalize()
-                dr.template_miner.config_filename =os.path.join(drain3_ini_location, 'drain3_no_masking.ini') #drain3_no_masking.ini'  #TODO fix the path relative
-                tm = dr.TemplateMiner()
+                from loglead.parsers import DrainTemplateMinerNoMasking as tm
                 self.df = self.df.with_columns(
                     drain=pl.col(field).map_elements(lambda x: tm.add_log_message(x), return_dtype=return_dtype))
 
@@ -141,14 +135,14 @@ def parse_drain(self, field = "e_message_normalized", drain_masking=False, repar
             # tm.drain.print_tree()
         return self.df 
     
-        
     def parse_brain(self, field = "e_message_normalized", reparse=False):
         self._handle_prerequisites([field])
         if reparse or "e_event_brain_id" not in self.df.columns:
             if "e_event_brain_id" in self.df.columns:
                 self.df = self.df.drop("e_event_brain_id")
-            import parsers.Brain.Brain as brain
-            brain_parser = brain.LogParser(messages=self.df[field])
+
+            from loglead.parsers import BrainParser
+            brain_parser = BrainParser(messages=self.df[field])
             brain_parser.parse() 
             df_new = brain_parser.df_log.select(pl.col("EventId").alias("e_event_brain_id"))
             self.df = pl.concat([self.df, df_new], how="horizontal")
@@ -159,8 +153,9 @@ def parse_ael(self,field = "e_message_normalized",  reparse=False):
         if reparse or "e_event_ael_id" not in self.df.columns:
             if "e_event_ael_id" in self.df.columns:
                 self.df = self.df.drop("e_event_ael_id")
-            import parsers.AEL.AEL as ael
-            ael_parser = ael.LogParser(messages=self.df[field])
+
+            from loglead.parsers import AELParser
+            ael_parser = AELParser(messages=self.df[field])
             ael_parser.parse() 
             df_new = ael_parser.df_log.select(pl.col("EventId").alias("e_event_ael_id"))
             self.df = pl.concat([self.df, df_new], how="horizontal")
@@ -210,12 +205,12 @@ def parse_iplom(self, field = "e_message_normalized", reparse=False, CT=0.35, PS
         if reparse or "e_event_iplom_id" not in self.df.columns:
             if "e_event_iplom_id" in self.df.columns:
                 self.df = self.df.drop("e_event_iplom_id")
-            import parsers.iplom.IPLoM as iplom
             if "row_nr" in self.df.columns:
                 self.df = self.df.drop("row_nr")
             self.df = self.df.with_row_count()
-            #TODO Storing each parser in self might eat a lot of memeory 
-            iplom_parser = iplom.LogParser(messages=self.df[field], CT=CT,PST=PST,lowerBound=lower_bound)#FST not implemented
+            from loglead.parsers import IPLoMParser
+            #TODO Storing each parser in self might eat a lot of memeory
+            iplom_parser = IPLoMParser(messages=self.df[field], CT=CT, PST=PST, lowerBound=lower_bound)#FST not implemented
             iplom_parser.parse()
             df_output = pl.DataFrame({
                 "row_nr": [row[0] for row in iplom_parser.output],
@@ -232,7 +227,6 @@ def parse_iplom(self, field = "e_message_normalized", reparse=False, CT=0.35, PS
             self.df = self.df.join(df_output, on="row_nr", how="left")
         return self.df
 
-
     #Faster version of IPLoM coming in 2024
     def parse_pliplom(self, field = "e_message_normalized",  reparse=False, CT=0.35, FST=0, PST=0,lower_bound=0.1, single_outlier_event=True):
         self._handle_prerequisites(["e_words"]) #Check word split method https://github.com/logpai/logparser/blob/main/logparser/IPLoM/IPLoM.py#L154
@@ -242,8 +236,8 @@ def parse_pliplom(self, field = "e_message_normalized",  reparse=False, CT=0.35,
             if "row_nr" in self.df.columns:
                 self.df = self.df.drop("row_nr")
             self.df = self.df.with_row_count()
-            import parsers.pl_iplom.pl_iplom as pl_iplom
-            pliplom_parser = pl_iplom.IPLoM(self.df, CT=CT, FST=FST, PST=PST,lower_bound=lower_bound, single_outlier_event=single_outlier_event)
+            from loglead.parsers import PL_IPLoMParser
+            pliplom_parser = PL_IPLoMParser(self.df, CT=CT, FST=FST, PST=PST, lower_bound=lower_bound, single_outlier_event=single_outlier_event)
             df_new = pliplom_parser.parse()
             #df_new = plimplom_parser.merge_partitions_to_dataframe()
             df_new = df_new.select([
@@ -257,8 +251,8 @@ def parse_pliplom(self, field = "e_message_normalized",  reparse=False, CT=0.35,
     def parse_lenma(self, field = "e_message_normalized",  reparse=False):
         self._handle_prerequisites(["e_words"])
         if reparse or "e_event_lenma_id" not in self.df.columns:
-            import parsers.lenma.lenma_template as lmt
-            lenma_tm = lmt.LenmaTemplateManager(threshold=0.9)
+            from loglead.parsers import LenmaTemplateManager
+            lenma_tm = LenmaTemplateManager(threshold=0.9)
             if "row_nr" in self.df.columns:
                 self.df = self.df.drop("row_nr")
             self.df = self.df.with_row_count()
@@ -286,10 +280,10 @@ def extract_id(obj):
     def parse_spell(self, field = "e_message_normalized",  reparse=False):
         self._handle_prerequisites([field])
         if reparse or "e_event_spell_id" not in self.df.columns:
-            from parsers.pyspell.spell import lcsmap
+            from loglead.parsers import SpellParser
             #if "e_message_normalized" not in self.df.columns:
             #    self.normalize()
-            spell = lcsmap(r'\s+')
+            spell = SpellParser(r'\s+')
             self.df = self.df.with_columns(
                 spell_obj=pl.col(field)
                     .map_elements(lambda x: spell.insert(x), return_dtype=pl.Object))
@@ -316,7 +310,7 @@ def create_neural_emb(self, field="e_message_normalized"):
         self._handle_prerequisites([field])
         if "e_bert_emb" not in self.df.columns:
             #Lazy import only if needed
-            from parsers.bert.bertembedding import BertEmbeddings
+            from loglead.parsers import BertEmbeddings
             #if "e_message_normalized" not in self.df.columns:
             #    self.normalize()
             self.bert_emb_gen = BertEmbeddings(bertmodel="albert")
@@ -401,7 +395,6 @@ def item_cumsum2(self, column="e_message_normalized", chronological_order=1, ano
 
         return self.df
 
-    
     def item_cumsum(self, column="e_message_normalized", chronological_order=True, ano_only=True, unique_only=True):
         self._handle_prerequisites([column, 'm_timestamp'])
         if ano_only:
@@ -431,131 +424,3 @@ def item_cumsum(self, column="e_message_normalized", chronological_order=True, a
 
         return self.df
 
-class SequenceEnhancer:
-    def __init__(self, df, df_seq):
-        self.df = df
-        self.df_seq = df_seq
-
-    def start_time(self):
-        df_temp = self.df.group_by('seq_id').agg(pl.col('m_timestamp').min().alias('start_time'))
-        self.df_seq = self.df_seq.join(df_temp, on='seq_id')
-        return self.df_seq
-
-    def end_time(self):
-        df_temp = self.df.group_by('seq_id').agg(pl.col('m_timestamp').max().alias('end_time'))
-        self.df_seq = self.df_seq.join(df_temp, on='seq_id')
-        return self.df_seq
-
-    def time_stamp(self):
-        #We used median as time stamp for sequence
-        df_temp = self.df.group_by('seq_id').agg(pl.col('m_timestamp').median().alias('time_stamp'))
-        self.df_seq = self.df_seq.join(df_temp, on='seq_id')
-        return self.df_seq
-
-
-    def seq_len(self):
-        # Count the number of rows for each seq_id
-        df_temp = self.df.group_by('seq_id').agg(pl.count().alias('seq_len'))
-        # Join this result with df_sequences on seq_id
-        self.df_seq = self.df_seq.join(df_temp, on='seq_id')
-        #Add an alias  that is compatible with the token len naming. 
-        #TODO Should be in events() function
-        self.df_seq = self.df_seq.with_columns(self.df_seq['seq_len'].alias('e_event_id_len'))
-
-        return self.df_seq
-
-    def events(self, event_col = "e_event_drain_id"):
-        # Aggregate event ids into a list for each seq_id
-        df_temp = self.df.group_by('seq_id').agg(pl.col(event_col).alias(event_col))
-        # Join this result with df_sequences on seq_id
-        self.df_seq = self.df_seq.join(df_temp, on='seq_id')
-        return self.df_seq
-
-
-    def tokens(self, token="e_words"):
-        #df_temp = self.df.group_by('seq_id').agg(pl.col(token).flatten().alias(token))
-        #Same as above but the above crashes due to out of memory problems. We might need this fix also in other rows
-        df_temp = self.df.select("seq_id", token).explode(token).group_by('seq_id').agg(pl.col(token))
-        # Join this result with df_sequences on seq_id
-        self.df_seq = self.df_seq.join(df_temp, on='seq_id')
-
-        #lengths
-        df_temp = self.df.group_by('seq_id').agg(pl.col(token+"_len").sum().alias(token+"_len"))
-        self.df_seq = self.df_seq.join(df_temp, on='seq_id')
-
-        return self.df_seq
-    
-    def duration(self):
-        # Calculate the sequence duration for each seq_id as the difference between max and min timestamps
-        df_temp = self.df.group_by('seq_id').agg(
-            (pl.col('m_timestamp').max() - pl.col('m_timestamp').min()).alias('duration'),
-            (pl.col('m_timestamp').max() - pl.col('m_timestamp').min()).dt.seconds().alias('duration_sec')
-        )
-        # Join this result with df_sequences on seq_id
-        self.df_seq = self.df_seq.join(df_temp, on='seq_id')
-        return self.df_seq
-
-    def eve_len(self):
-        # Count the number of rows for each seq_id
-        df_temp = self.df.group_by('seq_id').agg(
-            eve_len_max=pl.col('e_chars_len').max(),
-            eve_len_min=pl.col('e_chars_len').min(),
-            eve_len_avg=pl.col('e_chars_len').mean(),
-            eve_len_med=pl.col('e_chars_len').median(),
-            eve_len_over1=(pl.col('e_chars_len') > 1).sum()
-        )
-        # Join this result with df_sequences on seq_id
-        self.df_seq = self.df_seq.join(df_temp, on='seq_id')
-        return self.df_seq     
-    
-    def embeddings(self, embedding_column="e_bert_emb"):
-        # Aggregate by averaging the embeddings for each sequence (seq_id)
-        df_temp = self.df.select(pl.col("seq_id"), pl.col(embedding_column).list.to_struct()).unnest(embedding_column)
-        df_temp = df_temp.group_by('seq_id').mean()
-        df_temp = df_temp.select(pl.col("seq_id"),pl.concat_list(pl.col("*").exclude("seq_id")).alias(embedding_column))
-        # Join this result with df_sequences on seq_id
-        self.df_seq = self.df_seq.join(df_temp, on='seq_id')
-        return self.df_seq
-        
-    def next_event_prediction (self, event_col = "e_event_drain_id"):
-        if event_col not in self.df_seq.columns:#Ensure events are present otherwise no nep can be done
-            self.events(event_col)
-        nepn = nep.NextEventPredictionNgram()
-        #Create model and score with same data.
-        #This follows the enchancer logic that there is no splitting. In AD we operate with splits  
-        #Also sequence parsing is not done with splits so this follows the same logic
-        seq_data = self.df_seq[event_col].to_list()
-        nepn.create_ngram_model(seq_data)
-        #predicted events
-        preds, correct,  s_abs, spn_sum, spn_max = nepn.predict_list(seq_data)
-        self.df_seq = self.df_seq.with_columns(nep_predict = pl.Series(preds), 
-                                    nep_corr= pl.Series(correct),
-                                    nep_abs = pl.Series(s_abs),
-                                    nep_prob_nsum = pl.Series(spn_sum),
-                                    nep_prob_nmax = pl.Series(spn_max))
-        # Add average, max, and min columns for nep_corr
-        # This info is already in nep_prob_nmax with 1 being correct and <1 being zeros
-        #self.df_seq = self.df_seq.with_columns(pl.col("nep_corr").list.mean().alias("nep_corr_avg"))
-        #self.df_seq = self.df_seq.with_columns(pl.col("nep_corr").list.max().alias("nep_corr_max"))
-        #self.df_seq = self.df_seq.with_columns(pl.col("nep_corr").list.min().alias("nep_corr_min"))
-        # Add average, max, and min columns for nep_abs
-        self.df_seq = self.df_seq.with_columns(pl.col("nep_abs").list.mean().alias("nep_abs_avg"))
-        self.df_seq = self.df_seq.with_columns(pl.col("nep_abs").list.max().alias("nep_abs_max"))
-        self.df_seq = self.df_seq.with_columns(pl.col("nep_abs").list.min().alias("nep_abs_min"))
-        # Add average, max, and min columns for nep_prob_nsum
-        # For prediction it looks as nep_prob_nmax is superior
-        #self.df_seq = self.df_seq.with_columns(pl.col("nep_prob_nsum").list.mean().alias("nep_prob_nsum_avg"))
-        #self.df_seq = self.df_seq.with_columns(pl.col("nep_prob_nsum").list.max().alias("nep_prob_nsum_max"))
-        #self.df_seq = self.df_seq.with_columns(pl.col("nep_prob_nsum").list.min().alias("nep_prob_nsum_min"))
-        # Add average, max, and min columns for nep_prob_nmax
-        self.df_seq = self.df_seq.with_columns(pl.col("nep_prob_nmax").list.mean().alias("nep_prob_nmax_avg"))
-        self.df_seq = self.df_seq.with_columns(pl.col("nep_prob_nmax").list.max().alias("nep_prob_nmax_max"))
-        self.df_seq = self.df_seq.with_columns(pl.col("nep_prob_nmax").list.min().alias("nep_prob_nmax_min"))
-
-        self._perplexity(probab_col = "nep_prob_nmax")
-        return self.df_seq
-
-    def _perplexity(self, probab_col = "nep_prob_nmax"): 
-        self.df_seq = self.df_seq.with_columns(pl.col(probab_col).list.eval(pl.element().log()) #Log of probabilties in a list
-                                    .list.mean() #Average of probs
-                                    .mul(-1).exp().alias(probab_col + "_perp")) #flip sign and exponent
diff --git a/loglead/enhancers/sequence.py b/loglead/enhancers/sequence.py
new file mode 100644
index 0000000..c28b699
--- /dev/null
+++ b/loglead/enhancers/sequence.py
@@ -0,0 +1,133 @@
+import polars as pl
+
+from loglead import NextEventPredictionNgram
+
+__all__ = ['SequenceEnhancer']
+
+
+class SequenceEnhancer:
+    def __init__(self, df, df_seq):
+        self.df = df
+        self.df_seq = df_seq
+
+    def start_time(self):
+        df_temp = self.df.group_by('seq_id').agg(pl.col('m_timestamp').min().alias('start_time'))
+        self.df_seq = self.df_seq.join(df_temp, on='seq_id')
+        return self.df_seq
+
+    def end_time(self):
+        df_temp = self.df.group_by('seq_id').agg(pl.col('m_timestamp').max().alias('end_time'))
+        self.df_seq = self.df_seq.join(df_temp, on='seq_id')
+        return self.df_seq
+
+    def time_stamp(self):
+        # We used median as time stamp for sequence
+        df_temp = self.df.group_by('seq_id').agg(pl.col('m_timestamp').median().alias('time_stamp'))
+        self.df_seq = self.df_seq.join(df_temp, on='seq_id')
+        return self.df_seq
+
+    def seq_len(self):
+        # Count the number of rows for each seq_id
+        df_temp = self.df.group_by('seq_id').agg(pl.count().alias('seq_len'))
+        # Join this result with df_sequences on seq_id
+        self.df_seq = self.df_seq.join(df_temp, on='seq_id')
+        # Add an alias  that is compatible with the token len naming.
+        # TODO Should be in events() function
+        self.df_seq = self.df_seq.with_columns(self.df_seq['seq_len'].alias('e_event_id_len'))
+
+        return self.df_seq
+
+    def events(self, event_col="e_event_drain_id"):
+        # Aggregate event ids into a list for each seq_id
+        df_temp = self.df.group_by('seq_id').agg(pl.col(event_col).alias(event_col))
+        # Join this result with df_sequences on seq_id
+        self.df_seq = self.df_seq.join(df_temp, on='seq_id')
+        return self.df_seq
+
+    def tokens(self, token="e_words"):
+        # df_temp = self.df.group_by('seq_id').agg(pl.col(token).flatten().alias(token))
+        # Same as above but the above crashes due to out of memory problems. We might need this fix also in other rows
+        df_temp = self.df.select("seq_id", token).explode(token).group_by('seq_id').agg(pl.col(token))
+        # Join this result with df_sequences on seq_id
+        self.df_seq = self.df_seq.join(df_temp, on='seq_id')
+
+        # lengths
+        df_temp = self.df.group_by('seq_id').agg(pl.col(token+"_len").sum().alias(token+"_len"))
+        self.df_seq = self.df_seq.join(df_temp, on='seq_id')
+
+        return self.df_seq
+
+    def duration(self):
+        # Calculate the sequence duration for each seq_id as the difference between max and min timestamps
+        df_temp = self.df.group_by('seq_id').agg(
+            (pl.col('m_timestamp').max() - pl.col('m_timestamp').min()).alias('duration'),
+            (pl.col('m_timestamp').max() - pl.col('m_timestamp').min()).dt.seconds().alias('duration_sec')
+        )
+        # Join this result with df_sequences on seq_id
+        self.df_seq = self.df_seq.join(df_temp, on='seq_id')
+        return self.df_seq
+
+    def eve_len(self):
+        # Count the number of rows for each seq_id
+        df_temp = self.df.group_by('seq_id').agg(
+            eve_len_max=pl.col('e_chars_len').max(),
+            eve_len_min=pl.col('e_chars_len').min(),
+            eve_len_avg=pl.col('e_chars_len').mean(),
+            eve_len_med=pl.col('e_chars_len').median(),
+            eve_len_over1=(pl.col('e_chars_len') > 1).sum()
+        )
+        # Join this result with df_sequences on seq_id
+        self.df_seq = self.df_seq.join(df_temp, on='seq_id')
+        return self.df_seq
+
+    def embeddings(self, embedding_column="e_bert_emb"):
+        # Aggregate by averaging the embeddings for each sequence (seq_id)
+        df_temp = self.df.select(pl.col("seq_id"), pl.col(embedding_column).list.to_struct()).unnest(embedding_column)
+        df_temp = df_temp.group_by('seq_id').mean()
+        df_temp = df_temp.select(pl.col("seq_id"), pl.concat_list(pl.col("*").exclude("seq_id")).alias(embedding_column))
+        # Join this result with df_sequences on seq_id
+        self.df_seq = self.df_seq.join(df_temp, on='seq_id')
+        return self.df_seq
+
+    def next_event_prediction(self, event_col="e_event_drain_id"):
+        if event_col not in self.df_seq.columns:  # Ensure events are present otherwise no nep can be done
+            self.events(event_col)
+        nepn = NextEventPredictionNgram()
+        # Create model and score with same data.
+        # This follows the enhancer logic that there is no splitting. In AD we operate with splits
+        # Also sequence parsing is not done with splits so this follows the same logic
+        seq_data = self.df_seq[event_col].to_list()
+        nepn.create_ngram_model(seq_data)
+        # predicted events
+        preds, correct,  s_abs, spn_sum, spn_max = nepn.predict_list(seq_data)
+        self.df_seq = self.df_seq.with_columns(nep_predict=pl.Series(preds),
+                                               nep_corr=pl.Series(correct),
+                                               nep_abs=pl.Series(s_abs),
+                                               nep_prob_nsum=pl.Series(spn_sum),
+                                               nep_prob_nmax=pl.Series(spn_max))
+        # Add average, max, and min columns for nep_corr
+        # This info is already in nep_prob_nmax with 1 being correct and <1 being zeros
+        # self.df_seq = self.df_seq.with_columns(pl.col("nep_corr").list.mean().alias("nep_corr_avg"))
+        # self.df_seq = self.df_seq.with_columns(pl.col("nep_corr").list.max().alias("nep_corr_max"))
+        # self.df_seq = self.df_seq.with_columns(pl.col("nep_corr").list.min().alias("nep_corr_min"))
+        # Add average, max, and min columns for nep_abs
+        self.df_seq = self.df_seq.with_columns(pl.col("nep_abs").list.mean().alias("nep_abs_avg"))
+        self.df_seq = self.df_seq.with_columns(pl.col("nep_abs").list.max().alias("nep_abs_max"))
+        self.df_seq = self.df_seq.with_columns(pl.col("nep_abs").list.min().alias("nep_abs_min"))
+        # Add average, max, and min columns for nep_prob_nsum
+        # For prediction it looks as nep_prob_nmax is superior
+        # self.df_seq = self.df_seq.with_columns(pl.col("nep_prob_nsum").list.mean().alias("nep_prob_nsum_avg"))
+        # self.df_seq = self.df_seq.with_columns(pl.col("nep_prob_nsum").list.max().alias("nep_prob_nsum_max"))
+        # self.df_seq = self.df_seq.with_columns(pl.col("nep_prob_nsum").list.min().alias("nep_prob_nsum_min"))
+        # Add average, max, and min columns for nep_prob_nmax
+        self.df_seq = self.df_seq.with_columns(pl.col("nep_prob_nmax").list.mean().alias("nep_prob_nmax_avg"))
+        self.df_seq = self.df_seq.with_columns(pl.col("nep_prob_nmax").list.max().alias("nep_prob_nmax_max"))
+        self.df_seq = self.df_seq.with_columns(pl.col("nep_prob_nmax").list.min().alias("nep_prob_nmax_min"))
+
+        self._perplexity(probab_col="nep_prob_nmax")
+        return self.df_seq
+
+    def _perplexity(self, probab_col="nep_prob_nmax"):
+        self.df_seq = self.df_seq.with_columns(pl.col(probab_col).list.eval(pl.element().log()) #Log of probabilties in a list
+                                    .list.mean() #Average of probs
+                                    .mul(-1).exp().alias(probab_col + "_perp")) #flip sign and exponent
diff --git a/loglead/loaders/__init__.py b/loglead/loaders/__init__.py
index 19a60a2..6640c75 100644
--- a/loglead/loaders/__init__.py
+++ b/loglead/loaders/__init__.py
@@ -1,6 +1,13 @@
+from .adfa import ADFALoader
+from .awsctd import AWSCTDLoader
+from .base import BaseLoader
+from .bgl import BGLLoader
+from .gelf import GELFLoader
 from .hadoop import HadoopLoader
-from .supercomputers import ThuSpiLibLoader, BGLLoader
-from .pro import ProLoader
 from .hdfs import HDFSLoader
 from .nezha import NezhaLoader
-from .base import BaseLoader
\ No newline at end of file
+from .pro import ProLoader
+from .supercomputers import ThuSpiLibLoader
+
+__all__ = ['ADFALoader', 'AWSCTDLoader', 'BGLLoader', 'GELFLoader', 'HadoopLoader', 'HDFSLoader', 'NezhaLoader',
+           'ProLoader', 'ThuSpiLibLoader', 'BaseLoader']
\ No newline at end of file
diff --git a/loglead/loaders/adfa.py b/loglead/loaders/adfa.py
index b5d5de1..268d29f 100644
--- a/loglead/loaders/adfa.py
+++ b/loglead/loaders/adfa.py
@@ -1,8 +1,12 @@
-from loglead.loaders.base import BaseLoader
-import polars as pl
 from glob import glob
 import os
 
+import polars as pl
+
+from .base import BaseLoader
+
+__all__ = ['ADFALoader']
+
 
 class ADFALoader(BaseLoader):
     """
@@ -16,7 +20,6 @@ def __init__(self, filename, df=None, df_seq=None):
         super().__init__(filename, df, df_seq)
         self._mandatory_columns = ["m_message"]
 
-
     def load(self):
         m_messages = []
         seq_ids = []
diff --git a/loglead/loaders/awsctd.py b/loglead/loaders/awsctd.py
index 323e1bd..0c860c6 100644
--- a/loglead/loaders/awsctd.py
+++ b/loglead/loaders/awsctd.py
@@ -1,8 +1,13 @@
-from loglead.loaders.base import BaseLoader
-import polars as pl
 from glob import glob
 import os
 
+import polars as pl
+
+from .base import BaseLoader
+
+__all__ = ['AWSCTDLoader']
+
+
 class AWSCTDLoader(BaseLoader):
     """
     Note, that this dataset already consists of event IDs, so further enhancing is not required.
@@ -14,7 +19,6 @@ def __init__(self, filename, df=None, df_seq=None):
         super().__init__(filename, df, df_seq)
         self._mandatory_columns = ["m_message"]
 
-          
     def load(self):
         m_messages = []
         seq_ids = []
@@ -30,7 +34,8 @@ def load(self):
                     label = line_parts[-1]
                     if label == "Clean":
                         label = "Normal"
-                    seq_id = logfile_parts[-2] + '/' + logfile_parts[-1].replace('.csv', '') + '_' + str(line_nr)  # Use filename + incrementing id per sequence
+                    # Use filename + incrementing id per sequence
+                    seq_id = logfile_parts[-2] + '/' + logfile_parts[-1].replace('.csv', '') + '_' + str(line_nr)
 
                     for event_id in line_parts[:-1]:
                         # Append data to lists
diff --git a/loglead/loaders/base.py b/loglead/loaders/base.py
index ba747a7..26232eb 100644
--- a/loglead/loaders/base.py
+++ b/loglead/loaders/base.py
@@ -1,28 +1,28 @@
+import json
 
-#import glob
 import polars as pl
-#import os
-from collections import Counter
-import json
+
+__all__ = ['BaseLoader']
+
 
 # Base class
 class BaseLoader:
-    #This csv separator should never be found. 
-    #We try to disable polars from doing csv splitting.
-    #Instead we do it manually to get it correctly done. 
+    # This csv separator should never be found.
+    # We try to disable polars from doing csv splitting.
+    # Instead we do it manually to get it correctly done.
     _csv_separator = "\a" 
     _mandatory_columns = ["m_message", "m_timestamp"]
     
     def __init__(self, filename, df=None, df_seq=None):
         self.filename = filename
-        self.df = df #Event level dataframe
-        self.df_seq = df_seq #sequence level dataframe
+        self.df = df  # Event level dataframe
+        self.df_seq = df_seq  # Sequence level dataframe
 
     def load(self):
         print(f"WARNING! You are using dummy loader. This results in dataframe with single column only titled: m_message"
-              f"Consider implmenting dataset specific loader")
+              f"Consider implementing dataset specific loader")
         self.df = pl.read_csv(self.filename, has_header=False, infer_schema_length=0, 
-                        separator=self._csv_separator, ignore_errors=True)
+                              separator=self._csv_separator, ignore_errors=True)
         self.df = self.df.rename({"column_1": "m_message"})
         
     def preprocess(self):
@@ -39,7 +39,7 @@ def execute(self):
     
     def add_ano_col(self):
         # Check if the 'normal' column exists
-        if self.df is not None and  "normal" in self.df.columns:
+        if self.df is not None and "normal" in self.df.columns:
             # Create the 'anomaly' column by inverting the boolean values of the 'normal' column
             self.df = self.df.with_columns(pl.col("normal").not_().alias("anomaly"))
         if self.df_seq is not None and "normal" in self.df_seq:
@@ -47,12 +47,11 @@ def add_ano_col(self):
             self.df_seq = self.df_seq.with_columns(pl.col("normal").not_().alias("anomaly"))
 
         # Check if the 'anomaly' column exists but no normal column
-        if self.df is not None and  "anomaly" in self.df.columns and not "normal" in self.df.columns:
+        if self.df is not None and "anomaly" in self.df.columns and not "normal" in self.df.columns:
             # Create the 'normal' column by inverting the boolean values of the 'anomaly' column
             self.df = self.df.with_columns(pl.col("anomaly").not_().alias("normal"))
-        #self._mandatory_columns = ["m_message"]
+        # self._mandatory_columns = ["m_message"]
 
-    
     def check_for_nulls(self):
         null_counts = {}  # Dictionary to store count of nulls for each column
         for col in self.df.columns:
@@ -63,10 +62,10 @@ def check_for_nulls(self):
         if null_counts:
             for col, count in null_counts.items():
                 print(f"WARNING! Column '{col}' has {count} null values out of {len(self.df)}. You have 4 options:"
-                        f" 1) Do nothing and hope for the best"
-                        f", 2) Drop the column that has nulls"
-                        f", 3) Filter out rows that have nulls"
-                        f", 4) Investigate and fix your Loader")
+                      f" 1) Do nothing and hope for the best"
+                      f", 2) Drop the column that has nulls"
+                      f", 3) Filter out rows that have nulls"
+                      f", 4) Investigate and fix your Loader")
                 print(f"To investigate: <DF_NAME>.filter(<DF_NAME>['{col}'].is_null())")
 
     def check_mandatory_columns(self):
@@ -78,7 +77,7 @@ def check_mandatory_columns(self):
             raise TypeError("Column 'm_time_stamp' is not of type Polars.Datetime")
 
     def _split_and_unnest(self, field_names):
-        #split_cols = self.df["column_1"].str.splitn(" ", n=len(field_names))
+        # split_cols = self.df["column_1"].str.splitn(" ", n=len(field_names))
         split_cols = self.df.select(pl.col("column_1")).to_series().str.splitn(" ", n=len(field_names))
         split_cols = split_cols.struct.rename_fields(field_names)
         split_cols = split_cols.alias("fields")
@@ -106,7 +105,7 @@ def lines_not_starting_with_pattern(self, pattern=None):
     def reduce_dataframes(self, frac=0.5, random_state=42):
         # If df_sequences is present, reduce its size
         if hasattr(self, 'df_seq') and self.df_seq is not None:
-             # Sample df_seq
+            # Sample df_seq
             df_seq_temp = self.df_seq.sample(fraction=frac, seed=random_state)
 
             # Check if df_seq still has at least one row
@@ -118,37 +117,16 @@ def reduce_dataframes(self, frac=0.5, random_state=42):
             # Update df to include only the rows that have seq_id values present in the filtered df_seq
             self.df = self.df.filter(pl.col("seq_id").is_in(self.df_seq["seq_id"]))
 
-            #self.df_seq = self.df_seq.sample(fraction=frac)
+            # self.df_seq = self.df_seq.sample(fraction=frac)
             # Update df to include only the rows that have seq_id values present in the filtered df_sequences
-            #self.df = self.df.filter(pl.col("seq_id").is_in(self.df_seq["seq_id"]))
+            # self.df = self.df.filter(pl.col("seq_id").is_in(self.df_seq["seq_id"]))
         else:
             # If df_sequences is not present, just reduce df
             self.df = self.df.sample(fraction=frac, seed=random_state)
 
         return self.df
     
-    def parse_json(self, json_line):
+    @staticmethod
+    def parse_json(json_line):
         json_data = json.loads(json_line)
         return pl.DataFrame([json_data])
-
-#Process log files created with the GELF logging driver.
-# 
-class GELFLoader(BaseLoader):
-    def load(self):
-        with open(self.filename, 'r') as file:
-            lines = file.readlines()
-
-        # Assuming each line in the file is a separate JSON object
-        json_frames = [self.parse_json(line) for line in lines]
-        self.df = pl.concat(json_frames)
-
-    def preprocess(self):
-        # Rename some columns to match the expected column names and parse datetime
-        self.df = self.df.with_columns(
-            pl.col("message").alias("m_message")
-            ).drop("message")
-        
-        parsed_timestamps = self.df.select(
-            pl.col("@timestamp").str.strptime(pl.Datetime, strict=False).alias("m_timestamp")
-        ).drop("@timestamp")
-        self.df = self.df.with_columns(parsed_timestamps)
\ No newline at end of file
diff --git a/loglead/loaders/bgl.py b/loglead/loaders/bgl.py
new file mode 100644
index 0000000..c066473
--- /dev/null
+++ b/loglead/loaders/bgl.py
@@ -0,0 +1,20 @@
+import polars as pl
+
+from .base import BaseLoader
+
+__all__ = ['BGLLoader']
+
+
+# Processor for the BGL log file
+# At the moment there are 34470 null messages that are not handled by the loader
+class BGLLoader(BaseLoader):
+    def load(self):
+        self.df = pl.read_csv(self.filename, has_header=False, infer_schema_length=0,
+                              separator=self._csv_separator, ignore_errors=True)
+
+    def preprocess(self):
+        self._split_and_unnest(["label", "timestamp", "date", "node", "time",
+                                "noderepeat", "type", "component", "level", "m_message"])
+        self.df = self.df.with_columns(normal=pl.col("label").str.starts_with("-"))  # Same format as Tb
+        # Parse datatime
+        self.df = self.df.with_columns(m_timestamp=pl.from_epoch(pl.col("timestamp")))
diff --git a/loglead/loaders/gelf.py b/loglead/loaders/gelf.py
new file mode 100644
index 0000000..15990f4
--- /dev/null
+++ b/loglead/loaders/gelf.py
@@ -0,0 +1,28 @@
+import polars as pl
+
+from .base import BaseLoader
+
+__all__ = ['GELFLoader']
+
+
+# Process log files created with the GELF logging driver.
+#
+class GELFLoader(BaseLoader):
+    def load(self):
+        with open(self.filename, 'r') as file:
+            lines = file.readlines()
+
+        # Assuming each line in the file is a separate JSON object
+        json_frames = [self.parse_json(line) for line in lines]
+        self.df = pl.concat(json_frames)
+
+    def preprocess(self):
+        # Rename some columns to match the expected column names and parse datetime
+        self.df = self.df.with_columns(
+            pl.col("message").alias("m_message")
+            ).drop("message")
+
+        parsed_timestamps = self.df.select(
+            pl.col("@timestamp").str.strptime(pl.Datetime, strict=False).alias("m_timestamp")
+        ).drop("@timestamp")
+        self.df = self.df.with_columns(parsed_timestamps)
diff --git a/loglead/loaders/hadoop.py b/loglead/loaders/hadoop.py
index b412f66..3753c0a 100644
--- a/loglead/loaders/hadoop.py
+++ b/loglead/loaders/hadoop.py
@@ -1,8 +1,13 @@
-from loglead.loaders.base import BaseLoader
-import polars as pl
 import os
 import glob
 
+import polars as pl
+
+from .base import BaseLoader
+
+__all__ = ['HadoopLoader']
+
+
 class HadoopLoader(BaseLoader):
     def __init__(self, filename, df=None, df_seq=None, filename_pattern=None, labels_file_name=None):
         self.labels_file_name = labels_file_name
@@ -34,7 +39,8 @@ def load(self):
             # Iterate over all files in the subdirectory that match the given pattern
             for file in glob.glob(file_pattern):
                 try:
-                    q = pl.scan_csv(file, has_header=False, infer_schema_length=0, separator=self._csv_separator, row_count_name="row_nr_per_file")
+                    q = pl.scan_csv(file, has_header=False, infer_schema_length=0,
+                                    separator=self._csv_separator, row_count_name="row_nr_per_file")
                     q = q.with_columns(
                         pl.lit(seq_id).alias('seq_id'), #Folder is seq_id
                         pl.lit(os.path.basename(file)).alias('seq_id_sub') #File is seq_id_sub
@@ -126,8 +132,6 @@ def _parse_labels(self):
         })
         return label_df
     
-        
-        
     def _parse_datetimes(self):
         parsed_times = self.df.select(pl.concat_str([pl.col("date"), pl.col("time")]).alias("m_timestamp"))
 
diff --git a/loglead/loaders/hdfs.py b/loglead/loaders/hdfs.py
index 2e533af..dd2622c 100644
--- a/loglead/loaders/hdfs.py
+++ b/loglead/loaders/hdfs.py
@@ -1,6 +1,10 @@
-from loglead.loaders.base import BaseLoader
 import polars as pl
 
+from .base import BaseLoader
+
+__all__ = ['HDFSLoader']
+
+
 class HDFSLoader(BaseLoader):
     
     def __init__(self, filename, df=None, df_seq=None, labels_file_name=None):
@@ -11,11 +15,11 @@ def load(self):
         self.df = pl.read_csv(self.filename, has_header=False, infer_schema_length=0, separator=self._csv_separator)
 
     def preprocess(self):
-        #self._split_columns()
+        # self._split_columns()
         self._split_and_unnest(["date", "time", "id", "level", "component", "m_message"])
         self._extract_seq_id()
         self._parse_datetimes()
-        #Aggregate labels to sequence dataframe info that is at BlockID level
+        # Aggregate labels to sequence dataframe info that is at BlockID level
         self.df_seq = self.df.select(pl.col("seq_id")).unique()  
         df_temp = pl.read_csv(self.labels_file_name, has_header=True)
         self.df_seq = self.df_seq.join(df_temp, left_on='seq_id', right_on="BlockId")
@@ -25,11 +29,11 @@ def preprocess(self):
         self.df_seq = self.df_seq.drop("Label")
 
     def _extract_seq_id(self):
-        #seq_id = self.df.select(pl.col("m_message").str.extract(r"blk_(-?\d+)", group_index=1).alias("seq_id"))
+        # seq_id = self.df.select(pl.col("m_message").str.extract(r"blk_(-?\d+)", group_index=1).alias("seq_id"))
         seq_id = self.df.select(pl.col("m_message").str.extract(r"(blk_[-?\d]+)", group_index=1).alias("seq_id"))
         self.df = self.df.with_columns(seq_id)
 
     def _parse_datetimes(self):
         parsed_times = self.df.select(pl.concat_str([pl.col("date"), pl.col("time")]).alias("m_timestamp"))
         parsed_times = parsed_times.to_series().str.strptime(pl.Datetime, "%y%m%d%H%M%S")
-        self.df = self.df.with_columns(parsed_times)
\ No newline at end of file
+        self.df = self.df.with_columns(parsed_times)
diff --git a/loglead/loaders/nezha.py b/loglead/loaders/nezha.py
index 46be985..9c8295f 100644
--- a/loglead/loaders/nezha.py
+++ b/loglead/loaders/nezha.py
@@ -1,13 +1,17 @@
 import glob
 import os
-from loglead.loaders.base import BaseLoader
-import polars as pl
 import re
 import json
-
 import traceback
 import logging
+
+import polars as pl
+
+from loglead.loaders.base import BaseLoader
+
 logger = logging.getLogger(__name__)
+__all__ = ['NezhaLoader']
+
 
 class NezhaLoader(BaseLoader):
     def __init__(self, filename, system, df=None, df_seq=None):
@@ -28,7 +32,6 @@ def __init__(self, filename, system, df=None, df_seq=None):
             # If it's not, raise an error
             raise ValueError(f"Invalid system name: {system}. Valid options are: {', '.join(valid_options)}")
 
-
     def load(self):
         log_queries = []
         metric_queries = {
@@ -120,7 +123,6 @@ def load(self):
                         print(f"Columns in {group}: {collected_df.columns}")
                     raise
 
-
     def load_rca (self, file_path):
         # Read JSON data from a file
         with open(file_path, 'r') as file:
@@ -240,7 +242,6 @@ def preprocess(self):
         self.df_seq = self.add_labels_to_df_seq(self.df)
         self.df_trace_seq = self.add_labels_to_df_seq(self.df_trace)
 
-
     def process_metrics(self):            
         self.df_label = self.df_label.with_columns(
             (pl.col("m_timestamp") + pl.duration(minutes=1)).alias("m_timestamp+1"),
@@ -259,8 +260,6 @@ def process_metrics(self):
         for col in column_names:
             self.df_metric_default = self.df_metric_default.cast({col:pl.Float64})
 
-
-
     def _extract_log_message(self):
         self.df = self.df.with_row_count("row_key")#Used for matching later
         #Splitting criteria. We have valid json and invalid flag
@@ -545,11 +544,3 @@ def merge_logs_and_traces(self):
         self.df = self.df.with_columns(pl.col("seq_id").str.replace_all("_log.csv", ""))
         self.df_merge = self.df .join(self.df_trace, on="seq_id", how="inner")
         return self.df_merge
-
-# full_data = "/home/mmantyla/Datasets"
-# loader = NezhaLoader(filename= f"{full_data}/nezha/",) 
-# df = loader.execute()
-# loader.df.describe()
-# loader.df_trace.describe()
-# loader.df_metric_default.describe()
-
diff --git a/loglead/loaders/pro.py b/loglead/loaders/pro.py
index ef5a8bd..b0db8be 100644
--- a/loglead/loaders/pro.py
+++ b/loglead/loaders/pro.py
@@ -1,8 +1,13 @@
-from loglead.loaders.base import BaseLoader
-import polars as pl
 import glob
 import os
 
+import polars as pl
+
+from .base import BaseLoader
+
+__all__ = ['ProLoader']
+
+
 # Processor for the Pro log file - Not open dataset
 class ProLoader(BaseLoader):
     def load(self):
@@ -12,30 +17,30 @@ def load(self):
                 q = pl.scan_csv(file, has_header=False, infer_schema_length=0, separator=self._csv_separator)
                 q = q.with_columns((pl.lit(os.path.basename(file))).alias('seq_id'))
                 queries.append(q)
-            except pl.exceptions.NoDataError: # some CSV files can be empty.
+            except pl.exceptions.NoDataError:  # some CSV files can be empty.
                 continue
         dataframes = pl.collect_all(queries)
         self.df = pl.concat(dataframes)
 
     def preprocess(self):
         self._remove_extra_spaces()
-        #Store seq_id from files
+        # Store seq_id from files
         df_seq_id = self.df.select(pl.col("seq_id"))
         self._split_and_unnest(["count", "date", "time", "system", "nr1", "nr2", "log_level", "m_message"])
-        #Concat seq_id to main dataframe
+        # Concat seq_id to main dataframe
         self.df = pl.concat([self.df, df_seq_id], how="horizontal")  
         self._parse_datetimes()
-        #Dataframe for aggrating to sequence level
+        # Dataframe for aggrating to sequence level
         self.df_seq = self.df.select(pl.col("seq_id")).unique()   
-        #Filename holds the anomaly info. 
+        # Filename holds the anomaly info.
         self.df_seq = self.df_seq.with_columns(
             pl.col("seq_id").str.starts_with("success").alias("normal"),
         )
 
-    #Needed because multiple spaces and no regex support
-    #https://github.com/pola-rs/polars/issues/4819
+    # Needed because multiple spaces and no regex support
+    # https://github.com/pola-rs/polars/issues/4819
     def _remove_extra_spaces(self):
-        #df2 = self.df.with_columns(pl.col("column_1").str.extract_all(r"\S+").list.join(" ")) #Below is faster
+        # df2 = self.df.with_columns(pl.col("column_1").str.extract_all(r"\S+").list.join(" ")) #Below is faster
         df2 = self.df.with_columns(pl.col("column_1").str.replace_all(r"\s+", " "))         
         self.df = df2
 
diff --git a/loglead/loaders/supercomputers.py b/loglead/loaders/supercomputers.py
index 336b2d9..da37bde 100644
--- a/loglead/loaders/supercomputers.py
+++ b/loglead/loaders/supercomputers.py
@@ -1,16 +1,19 @@
-from loglead.loaders.base import BaseLoader
 import polars as pl
+
+from .base import BaseLoader
+
+__all__ = ['ThuSpiLibLoader']
+
+
 # Processor for the Thunderbird, Spirit and Liberty log files
 class ThuSpiLibLoader(BaseLoader):
     def __init__(self, filename, df=None, df_seq=None, split_component=True):
         self.split_component = split_component
         super().__init__(filename, df, df_seq)
 
-        
     def load(self):
         self.df = pl.read_csv(self.filename, has_header=False, infer_schema_length=0, 
-                              separator=self._csv_separator, ignore_errors=True) #There is one UTF error in the file
-    
+                              separator=self._csv_separator, ignore_errors=True)  # There is one UTF error in the file
     
     def preprocess(self):
         if self.split_component:
@@ -20,14 +23,14 @@ def preprocess(self):
         else:
             self._split_and_unnest(["label", "timestamp", "date", "userid", "month", 
                                     "day", "time", "location", "m_message"])
-        #parse datatime
-        self.df = self.df.with_columns(m_timestamp = pl.from_epoch(pl.col("timestamp")))
-        #Label contains multiple anomaly cases. Convert to binary
-        self.df = self.df.with_columns(normal = pl.col("label").str.starts_with("-"))
+        # parse datatime
+        self.df = self.df.with_columns(m_timestamp=pl.from_epoch(pl.col("timestamp")))
+        # Label contains multiple anomaly cases. Convert to binary
+        self.df = self.df.with_columns(normal=pl.col("label").str.starts_with("-"))
 
-    #Reason for extra processing. We want so separte pid from component and in the log file they are embedded
-    #Data description
-    #https://github.com/logpai/loghub/blob/master/Thunderbird/Thunderbird_2k.log_structured.csv     
+    # Reason for extra processing. We want so separte pid from component and in the log file they are embedded
+    # Data description
+    # https://github.com/logpai/loghub/blob/master/Thunderbird/Thunderbird_2k.log_structured.csv
     def _split_component_and_pid(self):
         component_and_pid = self.df.select(pl.col("component_pid")).to_series().str.splitn("[", n=2)
         component_and_pid = component_and_pid.struct.rename_fields(["component", "pid"])
@@ -36,21 +39,7 @@ def _split_component_and_pid(self):
         component_and_pid = component_and_pid.unnest("fields")
         component_and_pid = component_and_pid.with_columns(pl.col("component").str.rstrip(":"))
         component_and_pid = component_and_pid.with_columns(pl.col("pid").str.rstrip("]:"))
-        self.df= pl.concat([self.df, component_and_pid], how="horizontal")  
+        self.df = pl.concat([self.df, component_and_pid], how="horizontal")
         self.df = self.df.drop("component_pid")
         self.df = self.df.select(["label", "timestamp", "date", "userid", "month", 
-                                "day", "time", "location", "component","pid", "m_message"])
-
-# Processor for the BGL log file
-# At the moment there are 34470 null messages that are not handled by the loader
-class BGLLoader(BaseLoader):
-    def load(self):
-        self.df = pl.read_csv(self.filename, has_header=False, infer_schema_length=0, 
-                              separator=self._csv_separator, ignore_errors=True)
-    def preprocess(self):
-        self._split_and_unnest(["label", "timestamp", "date", "node", "time", 
-                            "noderepeat", "type", "component", "level", "m_message"])
-        self.df = self.df.with_columns(normal = pl.col("label").str.starts_with("-")) #Same format as Tb
-        #parse datatime
-        self.df = self.df.with_columns(m_timestamp = pl.from_epoch(pl.col("timestamp")))
-      
\ No newline at end of file
+                                  "day", "time", "location", "component", "pid", "m_message"])
diff --git a/loglead/next_event_prediction.py b/loglead/next_event_prediction.py
index 7c5dd78..3720ffd 100644
--- a/loglead/next_event_prediction.py
+++ b/loglead/next_event_prediction.py
@@ -1,37 +1,38 @@
-#This file implements next event prediction.
-#It is a slightly modified version from https://github.com/EvoTestOps/next_event_prediction
-#That was presented in paper
+# This file implements next event prediction.
+# It is a slightly modified version from https://github.com/EvoTestOps/next_event_prediction
+# That was presented in paper
 # Mäntylä M., Varela M., Hashemi S. 
-#"Pinpointing Anomaly Events in Logs from Stability Testing - N-Grams vs. Deep-Learning",
-#5th International Workshop on the Next Level of Test Automation (NEXTA), ICST Workshops, pp. 1-8, 2022, 
-#Preprint: https://arxiv.org/abs/2202.09214
-#Only N-gram version implemented here. For LSTM see link above. 
+# "Pinpointing Anomaly Events in Logs from Stability Testing - N-Grams vs. Deep-Learning",
+# 5th International Workshop on the Next Level of Test Automation (NEXTA), ICST Workshops, pp. 1-8, 2022,
+# Preprint: https://arxiv.org/abs/2202.09214
+# Only N-gram version implemented here. For LSTM see link above.
 
 
 from collections import Counter
-from collections import defaultdict  
+from collections import defaultdict
+
 import numpy as np
-import time
-import polars as pl
 
-#N-gram based next event prediction
-#Assumes parsed input e.g. via Drain parser 
-#Implemented on top of Python dictionaries with O(1) lookup time. 
-#Run in O(n) time 
+__all__ = ['NextEventPredictionNgram']
+
+
 class NextEventPredictionNgram:
+    """ N-gram based next event prediction
+        Assumes parsed input e.g. via Drain parser
+        Implemented on top of Python dictionaries with O(1) lookup time.
+        Runs in O(n) time
+    """
     
-    _start_ ="SoS" #Start of Sequence used in padding the sequence
-    _end_ = "EoS" #End of Sequence used in padding the sequence
+    _start_ = "SoS"  # Start of Sequence used in padding the sequence
+    _end_ = "EoS"  # End of Sequence used in padding the sequence
 
     def __init__(self, ngrams=5):
         self.n_gram_window_len = ngrams  # Length of window. Default is 5 (4+1: use 4 to predict 1)
-
         self.n_gram_counter = Counter()  # Counter for sequences length n_gram_window_len
         self.n_gram_counter_1 = Counter()  # Counter for sequences length n_gram_window_len-1
         self.n1_gram_dict = defaultdict()  # to keep mappings of possible following events e1 e2 -> e1 e2 e3, e1 e2 e4, 
         self.n1_gram_winner = dict()  # The event n following n-1 gram (the prediction)
 
-
     def create_ngram_model(self, train_data):
         ngrams = list()
         ngrams_minus_1 = list()
@@ -39,27 +40,26 @@ def create_ngram_model(self, train_data):
             n, n_minus_1 = self.slice_ngrams(seq)
             ngrams.extend(n)
             ngrams_minus_1.extend(n_minus_1)
-        self.n_gram_counter += Counter (ngrams)
-        self.n_gram_counter_1 += Counter (ngrams_minus_1)
+        self.n_gram_counter += Counter(ngrams)
+        self.n_gram_counter_1 += Counter(ngrams_minus_1)
 
         for idx, s in enumerate(ngrams):
             #dictionary for faster access from n-1 grams to n-grams, e.g. from  [e1 e2 e3] -> [e1 e2 e3 e4]; [e1 e2 e3] -> [e1 e2 e3 e5] etc...
-            self.n1_gram_dict.setdefault(ngrams_minus_1[idx],[]).append(s)
+            self.n1_gram_dict.setdefault(ngrams_minus_1[idx], []).append(s)
             #precompute the most likely sequence following n-1gram. Needed to keep prediction times fast
-            if (ngrams_minus_1[idx] in self.n1_gram_winner): #is there existing winner 
+            if ngrams_minus_1[idx] in self.n1_gram_winner: #is there existing winner
                 n_gram = self.n1_gram_winner[ngrams_minus_1[idx]]
-                if (self.n_gram_counter[n_gram] < self.n_gram_counter[s]): #there is but we are bigger replace
+                if self.n_gram_counter[n_gram] < self.n_gram_counter[s]: #there is but we are bigger replace
                     self.n1_gram_winner[ngrams_minus_1[idx]] = s
             else: 
                 self.n1_gram_winner[ngrams_minus_1[idx]] = s #no n-1-gram key or winner add a new one...
 
-
-    #Produce required n-grams. E.g. With sequence [e1 e2 e3 e4 e5] and n_gram_window_len=3 we produce [e1 e2 e3], [e2 e3 e4], and [e3 e4 5] 
-    def slice_ngrams (self, seq):
+    #Produce required n-grams. E.g. With sequence [e1 e2 e3 e4 e5] and n_gram_window_len=3 we produce [e1 e2 e3], [e2 e3 e4], and [e3 e4 5]
+    def slice_ngrams(self, seq):
         #Add SoS and EoS
         #with n-gram 3 it is SoS SoS E1 E2 E3 EoS
         #No need to pad more than one EoS as the final event to be predicted is EoS
-        seq = [self._start_]*(self.n_gram_window_len-1) +seq+[self._end_]
+        seq = [self._start_]*(self.n_gram_window_len-1) + seq + [self._end_]
         ngrams = list()
         ngrams_minus_1 = list()
         for i in range(self.n_gram_window_len, len(seq)+1):#len +1 because [0:i] leaves out the last element 
@@ -93,8 +93,6 @@ def predict_list(self, test_data):
         # Return all collected data
         return ngram_preds, ngram_preds_correct, scores_abs_list, scores_prop_norm_sum_list, scores_prop_norm_max_list
     
-
-
     # Return five nep results
     # 1. prediction = What was the predicted event (k=1): E1 E2 E3
     # 2. correct_preds = Binary 1 or 0 if the prediction is correct 1 1 0
diff --git a/parsers/AEL/AEL.py b/loglead/parsers/AEL/AEL.py
similarity index 94%
rename from parsers/AEL/AEL.py
rename to loglead/parsers/AEL/AEL.py
index 4cb89cb..4eb173f 100644
--- a/parsers/AEL/AEL.py
+++ b/loglead/parsers/AEL/AEL.py
@@ -15,17 +15,19 @@
 # =========================================================================
 
 
-import regex as re
-import os
 import hashlib
-import pandas as pd
-from datetime import datetime
+# from datetime import datetime
 from collections import defaultdict
 from functools import reduce
+
 import polars as pl
+import regex as re
+import pandas as pd
+
+__all__ = ['AELParser']
 
 
-class Event:
+class _Event:
     def __init__(self, logidx, Eventstr=""):
         self.id = hashlib.md5(Eventstr.encode("utf-8")).hexdigest()[0:8]
         self.logs = [logidx]
@@ -37,22 +39,22 @@ def refresh_id(self):
         self.id = hashlib.md5(self.Eventstr.encode("utf-8")).hexdigest()[0:8]
 
 
-class LogParser:
+class AELParser:
     def __init__(
         self,
         messages,
-        #indir,
-        #outdir,
-        #log_format,
+        # indir,
+        # outdir,
+        # log_format,
         minEventCount=2,
         merge_percent=1,
         rex=[],
         keep_para=True,
     ):
         self.messages = messages
-        #self.logformat = log_format
-        #self.path = indir
-        #self.savePath = outdir
+        # self.logformat = log_format
+        # self.path = indir
+        # self.savePath = outdir
         self.rex = rex
         self.minEventCount = minEventCount
         self.merge_percent = merge_percent
@@ -63,8 +65,8 @@ def __init__(
         self.keep_para = keep_para
 
     def parse(self, logname="Fakename"):
-        start_time = datetime.now()
-        #print("Parsing file: " + os.path.join(self.path, logname))
+        # start_time = datetime.now()
+        # print("Parsing file: " + os.path.join(self.path, logname))
         self.logname = logname
         self.load_data()
         self.tokenize()
@@ -109,7 +111,7 @@ def categorize(self):
                         event.logs.append(logidx)
                         break
                 if not matched:
-                    abin["Events"].append(Event(logidx, log))
+                    abin["Events"].append(_Event(logidx, log))
 
     def reconcile(self):
         """
@@ -184,7 +186,6 @@ def dump(self):
         self.df_log = pl.DataFrame(self.df_log)
         self.df_event = pl.DataFrame(self.df_event)
 
-
     def merge_event(self, e1, e2):
         for pos in range(len(e1.EventToken)):
             if e1.EventToken[pos] != e2.EventToken[pos]:
diff --git a/parsers/AEL/LICENSE b/loglead/parsers/AEL/LICENSE
similarity index 100%
rename from parsers/AEL/LICENSE
rename to loglead/parsers/AEL/LICENSE
diff --git a/parsers/AEL/README.md b/loglead/parsers/AEL/README.md
similarity index 98%
rename from parsers/AEL/README.md
rename to loglead/parsers/AEL/README.md
index ace1b6c..1b95dd8 100644
--- a/parsers/AEL/README.md
+++ b/loglead/parsers/AEL/README.md
@@ -1,70 +1,70 @@
-# AEL
-
-## LogLead integration note
-This code is exported from [LogParser](https://github.com/logpai/logparser/tree/main/logparser/AEL) project and integrated with minimal modification to LogLead. Majority of the original code that is not used in LogLead has been deleted. 
-
-One might find AEL Parameter settings that are optimized for different data sets usefull. They can be seen [here](https://github.com/logpai/logparser/blob/main/logparser/AEL/benchmark.py). 
-
-AEL relies heavily on pre-identified variables that are counted from <*> tokens. Such tokens are created with the regexes in the [file](https://github.com/logpai/logparser/blob/main/logparser/AEL/benchmark.py). 
-
-Original README.md text is retained in below, but it might be inaccurate in LogLead context. 
-
-## AEL
-AEL (Abstracting Execution Logs) is one of the state-of-the-art log parsing approaches, which comprises four steps: anonymize, tokenize, categorize, and reconcile. In particular, in the reconcile step, the original algorithm merges events that have only a different token. However, this process cannot handle the cases where one single template multiple different parameter tokens. To improve the generability of this algorithm, we use a parameter merge_percent to set the percentage of different tokens when merging two events.
-
-Read more information about AEL from the following paper:
-
-+ Zhen Ming Jiang, Ahmed E. Hassan, Gilbert Hamann, Parminder Flora. [An Automated Approach for Abstracting Execution Logs to Execution Events](http://www.cse.yorku.ca/~zmjiang/publications/jsme2008.pdf), *Journal of Software Maintenance and Evolution: Research and Practice (JSME)*, 2008.
-
-
-### Running
-
-The code has been tested in the following enviornment:
-+ python 3.7.6
-+ regex 2022.3.2
-+ pandas 1.0.1
-+ numpy 1.18.1
-+ scipy 1.4.1
-
-Run the following scripts to start the demo:
-
-```
-python demo.py
-```
-
-Run the following scripts to execute the benchmark:
-
-```
-python benchmark.py
-```
-
-### Benchmark
-
-Running the benchmark script on Loghub_2k datasets, you could obtain the following results.
-
-|   Dataset   | F1_measure | Accuracy |
-|:-----------:|:--------------|:------------|
-|     HDFS    | 0.999984       | 0.9975       |
-|    Hadoop   | 0.995869       | 0.869        |
-|    Spark    | 0.991018       | 0.905        |
-|  Zookeeper  | 0.995291       | 0.921        |
-|     BGL     | 0.999554       | 0.957        |
-|     HPC     | 0.992206       | 0.903        |
-| Thunderbird | 0.99913        | 0.941        |
-|   Windows   | 0.999281       | 0.6895       |
-|    Linux    | 0.99235        | 0.6725       |
-|   Android   | 0.940411       | 0.6815       |
-|  HealthApp  | 0.863969       | 0.5675       |
-|    Apache   | 1              | 1            |
-|  Proxifier  | 0.786336       | 0.495        |
-|   OpenSSH   | 0.987212       | 0.538        |
-|  OpenStack  | 0.986042       | 0.7575       |
-|     Mac     | 0.962113       | 0.7635       |
-
-
-### Citation
-
-:telescope: If you use our logparser tools or benchmarking results in your publication, please kindly cite the following papers.
-
-+ [**ICSE'19**] Jieming Zhu, Shilin He, Jinyang Liu, Pinjia He, Qi Xie, Zibin Zheng, Michael R. Lyu. [Tools and Benchmarks for Automated Log Parsing](https://arxiv.org/pdf/1811.03509.pdf). *International Conference on Software Engineering (ICSE)*, 2019.
-+ [**DSN'16**] Pinjia He, Jieming Zhu, Shilin He, Jian Li, Michael R. Lyu. [An Evaluation Study on Log Parsing and Its Use in Log Mining](https://jiemingzhu.github.io/pub/pjhe_dsn2016.pdf). *IEEE/IFIP International Conference on Dependable Systems and Networks (DSN)*, 2016.
+# AEL
+
+## LogLead integration note
+This code is exported from [LogParser](https://github.com/logpai/logparser/tree/main/logparser/AEL) project and integrated with minimal modification to LogLead. Majority of the original code that is not used in LogLead has been deleted. 
+
+One might find AEL Parameter settings that are optimized for different data sets usefull. They can be seen [here](https://github.com/logpai/logparser/blob/main/logparser/AEL/benchmark.py). 
+
+AEL relies heavily on pre-identified variables that are counted from <*> tokens. Such tokens are created with the regexes in the [file](https://github.com/logpai/logparser/blob/main/logparser/AEL/benchmark.py). 
+
+Original README.md text is retained in below, but it might be inaccurate in LogLead context. 
+
+## AEL
+AEL (Abstracting Execution Logs) is one of the state-of-the-art log parsing approaches, which comprises four steps: anonymize, tokenize, categorize, and reconcile. In particular, in the reconcile step, the original algorithm merges events that have only a different token. However, this process cannot handle the cases where one single template multiple different parameter tokens. To improve the generability of this algorithm, we use a parameter merge_percent to set the percentage of different tokens when merging two events.
+
+Read more information about AEL from the following paper:
+
++ Zhen Ming Jiang, Ahmed E. Hassan, Gilbert Hamann, Parminder Flora. [An Automated Approach for Abstracting Execution Logs to Execution Events](http://www.cse.yorku.ca/~zmjiang/publications/jsme2008.pdf), *Journal of Software Maintenance and Evolution: Research and Practice (JSME)*, 2008.
+
+
+### Running
+
+The code has been tested in the following enviornment:
++ python 3.7.6
++ regex 2022.3.2
++ pandas 1.0.1
++ numpy 1.18.1
++ scipy 1.4.1
+
+Run the following scripts to start the demo:
+
+```
+python demo.py
+```
+
+Run the following scripts to execute the benchmark:
+
+```
+python benchmark.py
+```
+
+### Benchmark
+
+Running the benchmark script on Loghub_2k datasets, you could obtain the following results.
+
+|   Dataset   | F1_measure | Accuracy |
+|:-----------:|:--------------|:------------|
+|     HDFS    | 0.999984       | 0.9975       |
+|    Hadoop   | 0.995869       | 0.869        |
+|    Spark    | 0.991018       | 0.905        |
+|  Zookeeper  | 0.995291       | 0.921        |
+|     BGL     | 0.999554       | 0.957        |
+|     HPC     | 0.992206       | 0.903        |
+| Thunderbird | 0.99913        | 0.941        |
+|   Windows   | 0.999281       | 0.6895       |
+|    Linux    | 0.99235        | 0.6725       |
+|   Android   | 0.940411       | 0.6815       |
+|  HealthApp  | 0.863969       | 0.5675       |
+|    Apache   | 1              | 1            |
+|  Proxifier  | 0.786336       | 0.495        |
+|   OpenSSH   | 0.987212       | 0.538        |
+|  OpenStack  | 0.986042       | 0.7575       |
+|     Mac     | 0.962113       | 0.7635       |
+
+
+### Citation
+
+:telescope: If you use our logparser tools or benchmarking results in your publication, please kindly cite the following papers.
+
++ [**ICSE'19**] Jieming Zhu, Shilin He, Jinyang Liu, Pinjia He, Qi Xie, Zibin Zheng, Michael R. Lyu. [Tools and Benchmarks for Automated Log Parsing](https://arxiv.org/pdf/1811.03509.pdf). *International Conference on Software Engineering (ICSE)*, 2019.
++ [**DSN'16**] Pinjia He, Jieming Zhu, Shilin He, Jian Li, Michael R. Lyu. [An Evaluation Study on Log Parsing and Its Use in Log Mining](https://jiemingzhu.github.io/pub/pjhe_dsn2016.pdf). *IEEE/IFIP International Conference on Dependable Systems and Networks (DSN)*, 2016.
diff --git a/parsers/Brain/Brain.py b/loglead/parsers/Brain/Brain.py
similarity index 98%
rename from parsers/Brain/Brain.py
rename to loglead/parsers/Brain/Brain.py
index 443292d..ad80cb5 100644
--- a/parsers/Brain/Brain.py
+++ b/loglead/parsers/Brain/Brain.py
@@ -15,10 +15,8 @@
 # limitations under the License.
 # =========================================================================
 
-from datetime import datetime
 from collections import Counter
-import os
-import pandas as pd
+
 import regex as re
 
 RED = "\033[31m"
@@ -26,7 +24,7 @@
 PINK = "\033[38;2;255;192;203m"
 
 
-class LogParser:
+class BrainParser:
     def __init__(
         self,
         messages,
@@ -68,7 +66,7 @@ def parse(self, logName="Fake_Name"):
 
         template_set = {}
         for key in group_len.keys():
-            Tree = tupletree(
+            Tree = _tupletree(
                 sorted_tuple_vector[key],
                 word_combinations[key],
                 word_combinations_reverse[key],
@@ -81,7 +79,7 @@ def parse(self, logName="Fake_Name"):
             parse_result = Tree.down_split(
                 root_set_detail_ID, self.threshold, root_set_detail
             )
-            template_set.update(output_result(parse_result))
+            template_set.update(_output_result(parse_result))
         #endtime = datetime.now()
         #print("Parsing done...")
         #print("Time taken   =   " + PINK + str(endtime - starttime) + RESET)
@@ -251,7 +249,7 @@ def get_frequecy_vector(self, sentences, filter, delimiter, dataset):
         return group_len, tuple_vector, frequency_vector
 
 
-class tupletree:
+class _tupletree:
     """
     tupletree(sorted_tuple_vector[key], word_combinations[key], word_combinations_reverse[key], tuple_vector[key], group_len[key])
 
@@ -375,7 +373,7 @@ def down_split(self, root_set_detail_ID, threshold, root_set_detail):
         return root_set_detail_ID
 
 
-def output_result(parse_result):
+def _output_result(parse_result):
     template_set = {}
     for key in parse_result.keys():
         for pr in parse_result[key]:
@@ -388,7 +386,7 @@ def output_result(parse_result):
                     template.append("<*>")
                     i += 1
                     continue
-                if exclude_digits(this):
+                if _exclude_digits(this):
                     template.append("<*>")
                     i += 1
                     continue
@@ -400,7 +398,7 @@ def output_result(parse_result):
 
 
 
-def exclude_digits(string):
+def _exclude_digits(string):
     """
     exclude the digits-domain words from partial constant
     """
diff --git a/parsers/Brain/LICENSE b/loglead/parsers/Brain/LICENSE
similarity index 100%
rename from parsers/Brain/LICENSE
rename to loglead/parsers/Brain/LICENSE
diff --git a/parsers/Brain/README.md b/loglead/parsers/Brain/README.md
similarity index 100%
rename from parsers/Brain/README.md
rename to loglead/parsers/Brain/README.md
diff --git a/parsers/README.md b/loglead/parsers/README.md
similarity index 100%
rename from parsers/README.md
rename to loglead/parsers/README.md
diff --git a/loglead/parsers/__init__.py b/loglead/parsers/__init__.py
new file mode 100644
index 0000000..8dce8f9
--- /dev/null
+++ b/loglead/parsers/__init__.py
@@ -0,0 +1,14 @@
+__all__ = ['AELParser', 'BrainParser', 'IPLoMParser', 'LenmaTemplateManager', 'PL_IPLoMParser',
+           'SpellParser', 'DrainTemplateMiner', 'DrainTemplateMinerNoMasking']
+from .AEL.AEL import AELParser
+try:
+    from .bert.bertembedding import BertEmbeddings
+    __all__.append('BertEmbeddings')
+except Exception as e:
+    pass
+from .Brain.Brain import BrainParser
+from .drain3.drain import DrainTemplateMiner, DrainTemplateMinerNoMasking
+from .iplom.IPLoM import IPLoMParser
+from .lenma.lenma import LenmaTemplateManager
+from .pl_iplom.pl_iplom import PL_IPLoMParser
+from .pyspell.spell import SpellParser
diff --git a/parsers/bert/README.md b/loglead/parsers/bert/README.md
similarity index 100%
rename from parsers/bert/README.md
rename to loglead/parsers/bert/README.md
diff --git a/parsers/bert/bertembedding.py b/loglead/parsers/bert/bertembedding.py
similarity index 93%
rename from parsers/bert/bertembedding.py
rename to loglead/parsers/bert/bertembedding.py
index 7c1d25c..ce52edb 100644
--- a/parsers/bert/bertembedding.py
+++ b/loglead/parsers/bert/bertembedding.py
@@ -1,79 +1,82 @@
-import tensorflow as tf
-from transformers import AlbertTokenizer, TFAlbertModel
-from transformers import BertTokenizer, TFBertModel
-import time
-
-
-class BertEmbeddings:
-    def __init__(self, bertmodel = "basebert"):
-
-        self.basebert = bertmodel
-        
-        #Print out all GPU and CPU devices
-        devices = tf.config.list_physical_devices()
-        print("All physical devices: ", devices)
-
-        gpus = tf.config.list_physical_devices('GPU')
-        print("GPUs: ", gpus)
-
-        cpus = tf.config.list_physical_devices('CPU')
-        print("CPUs: ", cpus)
-
-        if self.basebert == 'basebert':
-            self.tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
-            self.model = TFBertModel.from_pretrained('bert-base-uncased', output_hidden_states=True)
-            print("Using basebert")
-
-        if self.basebert == 'albert':
-            self.tokenizer = AlbertTokenizer.from_pretrained('albert-base-v2')
-            self.model = TFAlbertModel.from_pretrained('albert-base-v2', output_hidden_states=True)
-            print("Using albert")
-
-    def create_bert_emb(self, sentences):
-        #length in word piece tokens
-        max_length = 30
-        # Set cache batch size depending on GPU memory
-        cache_size = 1000
-        embeddings = []
-
-        # Start tracking time
-        start_time = time.time()
-
-        for i in range(0, len(sentences), cache_size):
-            batch_sentences = sentences[i: i + cache_size]
-
-            # Tokenize sentences and convert to tensor
-            tokenized_batch = self.tokenizer(batch_sentences, truncation=True, padding='max_length',
-                                             add_special_tokens=True, return_tensors='tf', max_length=max_length)
-
-            # Put the model in "evaluation" mode, meaning feed-forward operation.
-            self.model.trainable = False
-
-            inputs = {'input_ids': tokenized_batch['input_ids'],
-                      'attention_mask': tokenized_batch['attention_mask']}
-            outputs = self.model(inputs)
-
-            if self.basebert == 'basebert':
-                hidden_states = outputs[2]
-                token_vecs = hidden_states[-1]
-
-            if self.basebert == 'albert':
-                hidden_states = outputs['hidden_states']
-                # The last layer is at -1 index. Each layer contains embeddings for all tokens.
-                token_vecs = hidden_states[-1]
-
-            # Calculate the mean along the sequence length dimension for each sentence
-            sentence_embs = tf.reduce_mean(token_vecs, axis=1)
-
-            embeddings.append(sentence_embs)
-
-        # Concatenate all batch embeddings
-        embeddings = tf.concat(embeddings, axis=0)
-
-        # End tracking time
-        end_time = time.time()
-
-        # Calculate and print the time taken in seconds
-        elapsed_time = end_time - start_time
-        print(f"Time taken: {elapsed_time:.2f} seconds")
-        return embeddings
+import time
+
+import tensorflow as tf
+from transformers import AlbertTokenizer, TFAlbertModel
+from transformers import BertTokenizer, TFBertModel
+
+__all__ = ['BertEmbeddings']
+
+
+class BertEmbeddings:
+    def __init__(self, bertmodel="basebert"):
+
+        self.basebert = bertmodel
+        
+        # Print out all GPU and CPU devices
+        devices = tf.config.list_physical_devices()
+        print("All physical devices: ", devices)
+
+        gpus = tf.config.list_physical_devices('GPU')
+        print("GPUs: ", gpus)
+
+        cpus = tf.config.list_physical_devices('CPU')
+        print("CPUs: ", cpus)
+
+        if self.basebert == 'basebert':
+            self.tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
+            self.model = TFBertModel.from_pretrained('bert-base-uncased', output_hidden_states=True)
+            print("Using basebert")
+
+        if self.basebert == 'albert':
+            self.tokenizer = AlbertTokenizer.from_pretrained('albert-base-v2')
+            self.model = TFAlbertModel.from_pretrained('albert-base-v2', output_hidden_states=True)
+            print("Using albert")
+
+    def create_bert_emb(self, sentences):
+        # length in word piece tokens
+        max_length = 30
+        # Set cache batch size depending on GPU memory
+        cache_size = 1000
+        embeddings = []
+
+        # Start tracking time
+        start_time = time.time()
+
+        for i in range(0, len(sentences), cache_size):
+            batch_sentences = sentences[i: i + cache_size]
+
+            # Tokenize sentences and convert to tensor
+            tokenized_batch = self.tokenizer(batch_sentences, truncation=True, padding='max_length',
+                                             add_special_tokens=True, return_tensors='tf', max_length=max_length)
+
+            # Put the model in "evaluation" mode, meaning feed-forward operation.
+            self.model.trainable = False
+
+            inputs = {'input_ids': tokenized_batch['input_ids'],
+                      'attention_mask': tokenized_batch['attention_mask']}
+            outputs = self.model(inputs)
+
+            if self.basebert == 'basebert':
+                hidden_states = outputs[2]
+                token_vecs = hidden_states[-1]
+
+            if self.basebert == 'albert':
+                hidden_states = outputs['hidden_states']
+                # The last layer is at -1 index. Each layer contains embeddings for all tokens.
+                token_vecs = hidden_states[-1]
+
+            # Calculate the mean along the sequence length dimension for each sentence
+            sentence_embs = tf.reduce_mean(token_vecs, axis=1)
+
+            embeddings.append(sentence_embs)
+
+        # Concatenate all batch embeddings
+        embeddings = tf.concat(embeddings, axis=0)
+
+        # End tracking time
+        end_time = time.time()
+
+        # Calculate and print the time taken in seconds
+        elapsed_time = end_time - start_time
+        print(f"Time taken: {elapsed_time:.2f} seconds")
+        return embeddings
diff --git a/parsers/drain3/License.txt b/loglead/parsers/drain3/License.txt
similarity index 100%
rename from parsers/drain3/License.txt
rename to loglead/parsers/drain3/License.txt
diff --git a/parsers/drain3/README.md b/loglead/parsers/drain3/README.md
similarity index 100%
rename from parsers/drain3/README.md
rename to loglead/parsers/drain3/README.md
diff --git a/loglead/parsers/drain3/drain.py b/loglead/parsers/drain3/drain.py
new file mode 100644
index 0000000..53702c1
--- /dev/null
+++ b/loglead/parsers/drain3/drain.py
@@ -0,0 +1,18 @@
+import os.path
+
+from drain3 import TemplateMiner
+from drain3.template_miner_config import TemplateMinerConfig
+
+__all__ = ['DrainTemplateMiner', 'DrainTemplateMinerNoMasking']
+
+current_script_path = os.path.abspath(__file__)
+current_script_directory = os.path.dirname(current_script_path)
+ini_location = os.path.join(current_script_directory, 'drain3.ini')
+tmc = TemplateMinerConfig()
+tmc.load(ini_location)
+DrainTemplateMiner = TemplateMiner(config=tmc)
+
+ini_location = os.path.join(current_script_directory, 'drain3_no_masking.ini')
+tmc = TemplateMinerConfig()
+tmc.load(ini_location)
+DrainTemplateMinerNoMasking = TemplateMiner(config=tmc)
diff --git a/parsers/drain3/drain3.ini b/loglead/parsers/drain3/drain3.ini
similarity index 100%
rename from parsers/drain3/drain3.ini
rename to loglead/parsers/drain3/drain3.ini
diff --git a/parsers/drain3/drain3_no_masking.ini b/loglead/parsers/drain3/drain3_no_masking.ini
similarity index 100%
rename from parsers/drain3/drain3_no_masking.ini
rename to loglead/parsers/drain3/drain3_no_masking.ini
diff --git a/parsers/iplom/IPLoM.py b/loglead/parsers/iplom/IPLoM.py
similarity index 97%
rename from parsers/iplom/IPLoM.py
rename to loglead/parsers/iplom/IPLoM.py
index 0d8b989..b5c67d7 100644
--- a/parsers/iplom/IPLoM.py
+++ b/loglead/parsers/iplom/IPLoM.py
@@ -28,6 +28,7 @@
 logger = logging.getLogger(__name__)
 
 
+
 #Bugs in logparser Iplom implementation. To be reported. "Fixed" in this file
 #No check that partitionL has IDs that are long enough. Default is 200. So if event is 201 long this will fail
 #https://github.com/logpai/logparser/blob/main/logparser/IPLoM/IPLoM.py#L163
@@ -35,7 +36,7 @@
 #https://github.com/logpai/logparser/blob/main/logparser/IPLoM/IPLoM.py#L437C12-L437C31 
 # https://github.com/logpai/logparser/blob/main/logparser/IPLoM/IPLoM.py#L477  
 
-class Partition:
+class _Partition:
     """Wrap around the logs and the step number"""
 
     def __init__(self, stepNo, numOfLogs=0, lenOfLogs=0):
@@ -46,7 +47,7 @@ def __init__(self, stepNo, numOfLogs=0, lenOfLogs=0):
         self.lenOfLogs = lenOfLogs
 
 
-class Event:
+class _Event:
     def __init__(self, eventStr):
         self.eventStr = eventStr
         if len(eventStr) == 1 and eventStr[0] == "Outlier": #Keeping the outlier event readable. 
@@ -56,7 +57,7 @@ def __init__(self, eventStr):
         self.eventCount = 0
 
 
-class Para:
+class _Para:
     """Para class
 
     Attributes
@@ -96,7 +97,7 @@ def __init__(
 #       self.logformat = log_format
 
 
-class LogParser:
+class IPLoMParser:
     def __init__(
         self,
         messages,
@@ -113,7 +114,7 @@ def __init__(
         keep_para=True,
 
     ):
-        self.para = Para(
+        self.para = _Para(
 #            log_format=log_format,
             indir=indir,
             outdir=outdir,
@@ -136,7 +137,7 @@ def __init__(
 
         # Initialize some partitions which contain logs with different length
         for logLen in range(self.para.maxEventLen + 1):
-            self.partitionsL.append(Partition(stepNo=1, numOfLogs=0, lenOfLogs=logLen))
+            self.partitionsL.append(_Partition(stepNo=1, numOfLogs=0, lenOfLogs=logLen))
 
 
     def parse(self, logname="FakeName"):
@@ -250,7 +251,7 @@ def Step2(self):
                     self.partitionsL[0].logLL += logDLL[key]
                     self.partitionsL[0].numOfLogs += len(logDLL[key])
                 else:
-                    newPartition = Partition(
+                    newPartition = _Partition(
                         stepNo=2,
                         numOfLogs=len(logDLL[key]),
                         lenOfLogs=partition.lenOfLogs,
@@ -360,7 +361,7 @@ def Step3(self):
 
             newPartitionsD = {}
             if partition.stepNo == 2:
-                newPartitionsD["dumpKeyforMMrelationInStep2__"] = Partition(
+                newPartitionsD["dumpKeyforMMrelationInStep2__"] = _Partition(
                     stepNo=3, numOfLogs=0, lenOfLogs=partition.lenOfLogs
                 )
             # Split partition
@@ -368,7 +369,7 @@ def Step3(self):
                 # If is 1-1
                 if logL[p1] in oneToOneS:
                     if logL[p1] not in newPartitionsD:
-                        newPartitionsD[logL[p1]] = Partition(
+                        newPartitionsD[logL[p1]] = _Partition(
                             stepNo=3, numOfLogs=0, lenOfLogs=partition.lenOfLogs
                         )
                     newPartitionsD[logL[p1]].logLL.append(logL)
@@ -382,14 +383,14 @@ def Step3(self):
                     )
                     if split_rank == 1:
                         if logL[p1] not in newPartitionsD:
-                            newPartitionsD[logL[p1]] = Partition(
+                            newPartitionsD[logL[p1]] = _Partition(
                                 stepNo=3, numOfLogs=0, lenOfLogs=partition.lenOfLogs
                             )
                         newPartitionsD[logL[p1]].logLL.append(logL)
                         newPartitionsD[logL[p1]].numOfLogs += 1
                     else:
                         if logL[p2] not in newPartitionsD:
-                            newPartitionsD[logL[p2]] = Partition(
+                            newPartitionsD[logL[p2]] = _Partition(
                                 stepNo=3, numOfLogs=0, lenOfLogs=partition.lenOfLogs
                             )
                         newPartitionsD[logL[p2]].logLL.append(logL)
@@ -402,14 +403,14 @@ def Step3(self):
                     )
                     if split_rank == 1:
                         if logL[p1] not in newPartitionsD:
-                            newPartitionsD[logL[p1]] = Partition(
+                            newPartitionsD[logL[p1]] = _Partition(
                                 stepNo=3, numOfLogs=0, lenOfLogs=partition.lenOfLogs
                             )
                         newPartitionsD[logL[p1]].logLL.append(logL)
                         newPartitionsD[logL[p1]].numOfLogs += 1
                     else:
                         if logL[p2] not in newPartitionsD:
-                            newPartitionsD[logL[p2]] = Partition(
+                            newPartitionsD[logL[p2]] = _Partition(
                                 stepNo=3, numOfLogs=0, lenOfLogs=partition.lenOfLogs
                             )
                         newPartitionsD[logL[p2]].logLL.append(logL)
@@ -425,14 +426,14 @@ def Step3(self):
                     else:
                         if len(p1Set) < len(p2Set):
                             if logL[p1] not in newPartitionsD:
-                                newPartitionsD[logL[p1]] = Partition(
+                                newPartitionsD[logL[p1]] = _Partition(
                                     stepNo=3, numOfLogs=0, lenOfLogs=partition.lenOfLogs
                                 )
                             newPartitionsD[logL[p1]].logLL.append(logL)
                             newPartitionsD[logL[p1]].numOfLogs += 1
                         else:
                             if logL[p2] not in newPartitionsD:
-                                newPartitionsD[logL[p2]] = Partition(
+                                newPartitionsD[logL[p2]] = _Partition(
                                     stepNo=3, numOfLogs=0, lenOfLogs=partition.lenOfLogs
                                 )
                             newPartitionsD[logL[p2]].logLL.append(logL)
@@ -462,7 +463,7 @@ def Step4(self):
         #Bug fix
         #if self.para.PST == 0 and self.partitionsL[0].numOfLogs != 0:
         if self.para.PST != 0 and self.partitionsL[0].numOfLogs != 0:
-            event = Event(["Outlier"])
+            event = _Event(["Outlier"])
             event.eventCount = self.partitionsL[0].numOfLogs
             self.eventsL.append(event)
 
@@ -492,7 +493,7 @@ def Step4(self):
                 else:
                     e[columnIdx] = "<*>"
 
-            event = Event(e)
+            event = _Event(e)
             event.eventCount = partition.numOfLogs
 
             self.eventsL.append(event)
diff --git a/parsers/iplom/LICENSE b/loglead/parsers/iplom/LICENSE
similarity index 100%
rename from parsers/iplom/LICENSE
rename to loglead/parsers/iplom/LICENSE
diff --git a/parsers/iplom/README.md b/loglead/parsers/iplom/README.md
similarity index 100%
rename from parsers/iplom/README.md
rename to loglead/parsers/iplom/README.md
diff --git a/parsers/lenma/LICENSE b/loglead/parsers/lenma/LICENSE
similarity index 100%
rename from parsers/lenma/LICENSE
rename to loglead/parsers/lenma/LICENSE
diff --git a/parsers/lenma/README.md b/loglead/parsers/lenma/README.md
similarity index 100%
rename from parsers/lenma/README.md
rename to loglead/parsers/lenma/README.md
diff --git a/parsers/lenma/lenma_template.py b/loglead/parsers/lenma/lenma.py
similarity index 64%
rename from parsers/lenma/lenma_template.py
rename to loglead/parsers/lenma/lenma.py
index d2f56ca..9b396d9 100644
--- a/parsers/lenma/lenma_template.py
+++ b/loglead/parsers/lenma/lenma.py
@@ -10,10 +10,75 @@
 from sklearn.metrics import accuracy_score
 from sklearn.metrics.pairwise import cosine_similarity
 
-from .template import Template, TemplateManager
+__all__ = ['LenmaTemplateManager']
 
 
-class LenmaTemplate(Template):
+class _Template(object):
+    def __init__(self, index, words, logid):
+        self._index = index
+        self._words = words
+        self._nwords = len(words)
+        self._counts = 1
+        self._logid = [logid]
+
+    @property
+    def index(self):
+        return self._index
+
+    @property
+    def words(self):
+        return self._words
+
+    @property
+    def nwords(self):
+        return self._nwords
+
+    @property
+    def counts(self):
+        return self._counts
+
+    def _dump_as_json(self):
+        """Dumps the data structure as a JSON format to serialize the
+        object.
+
+        This internal function is called by the TemplateManager
+        class.
+        """
+        assert (False)
+
+    def _restore_from_json(self, data):
+        """Initializes the instance with the provided JSON data.
+
+        This internal function is normally called by the initializer.
+        """
+        assert (False)
+
+    def get_similarity_score(self, new_words):
+        """Retruens a similarity score.
+
+        Args:
+          new_words: An array of words.
+
+        Returns:
+          score: in float.
+        """
+        assert (False)
+
+    def update(self, new_words):
+        """Updates the template data using the supplied new_words.
+        """
+        assert (False)
+
+    def __str__(self):
+        template = ' '.join([self.words[idx] if self.words[idx] != '' else '*' for idx in range(self.nwords)])
+        return '{index}({nwords})({counts}):{template}'.format(
+            index=self.index,
+            nwords=self.nwords,
+            counts=self._counts,
+            template=' '.join([self.words[idx] if self.words[idx] != '' else '*' for idx in range(self.nwords)]))
+
+
+class _LenmaTemplate(_Template):
     def __init__(self, index=None, words=None, logid=None, json=None):
         if json is not None:
             # restore from the jsonized data.
@@ -150,7 +215,71 @@ def get_logids(self):
         return self._logid
 
 
-class LenmaTemplateManager(TemplateManager):
+class _TemplateManager(object):
+    def __init__(self):
+        self._templates = []
+
+    @property
+    def templates(self):
+        return self._templates
+
+    def infer_template(self, words):
+        """Infer the best matching template, or create a new template if there
+        is no similar template exists.
+
+        Args:
+          words: An array of words.
+
+        Returns:
+          A template instance.
+
+        """
+        assert (False)
+
+    def dump_template(self, index):
+        """Dumps a specified template data structure usually in a text
+        format.
+
+        Args:
+          index: a template index.
+
+        Returns:
+          A serialized text data of the specified template.
+        """
+        assert (False)
+
+    def restore_template(self, data):
+        """Creates a template instance from data (usually a serialized
+        data when LogDatabase.close() method is called.
+
+        This function is called by the LogDatabase class.
+
+        Args:
+          data: a data required to rebuild a template instance.
+
+        Returns:
+          A template instance.
+        """
+        assert (False)
+
+    def _append_template(self, template):
+        """Append a template.
+
+        This internal function may be called by the LogDatabase
+        class too.
+
+        Args:
+          template: a new template to be appended.
+
+        Returns:
+          template: the appended template.
+        """
+        assert (template.index == len(self.templates))
+        self.templates.append(template)
+        return template
+
+
+class LenmaTemplateManager(_TemplateManager):
     def __init__(self,
                  threshold=0.9,
                  predefined_templates=None):
@@ -164,7 +293,7 @@ def dump_template(self, index):
         return self.templates[index]._dump_as_json()
 
     def restore_template(self, data):
-        return LenmaTemplate(json=data)
+        return _LenmaTemplate(json=data)
 
     def infer_template(self, words, logid):
         nwords = len(words)
@@ -185,6 +314,6 @@ def infer_template(self, words, logid):
             return self.templates[index]
 
         new_template = self._append_template(
-            LenmaTemplate(len(self.templates), words, logid))
+            _LenmaTemplate(len(self.templates), words, logid))
         return new_template
 
diff --git a/parsers/pl_iplom/README.md b/loglead/parsers/pl_iplom/README.md
similarity index 100%
rename from parsers/pl_iplom/README.md
rename to loglead/parsers/pl_iplom/README.md
diff --git a/parsers/pl_iplom/pl_iplom.py b/loglead/parsers/pl_iplom/pl_iplom.py
similarity index 97%
rename from parsers/pl_iplom/pl_iplom.py
rename to loglead/parsers/pl_iplom/pl_iplom.py
index d1e9524..a99c599 100644
--- a/parsers/pl_iplom/pl_iplom.py
+++ b/loglead/parsers/pl_iplom/pl_iplom.py
@@ -1,11 +1,14 @@
-
-import polars as pl 
 import time
 import logging
+
+import polars as pl
+
 logger = logging.getLogger(__name__)
 
+__all__ = ['PL_IPLoMParser']
 
-class Partition:
+
+class _Partition:
     def __init__(self, df, len, split_trace):
         self.df = df
         #self.has_subpartitions = False
@@ -36,15 +39,13 @@ def create_template(self):
         #logger.info (f"Template created: {self.template}")
 
 
-
-
 #TODO 
 # 1) Log Template creation, DONE
 # 2) check why we get no trace S1 S3. We should have atleast some        
 # 2) Log Id Creation, 
 # 3) Rolling log IDs and templates back to Enhancer
 # how it is done on tipping
-class IPLoM:
+class PL_IPLoMParser:
     def __init__(self, df, CT = 0.35, PST=0.001, FST=0.00001,lower_bound = 0.25, single_outlier_event = True):
         self.df = df
         self.CT = CT #Cluster goodness threshold
@@ -199,7 +200,6 @@ def parse(self):
         logger.info (f"Merge done in {time_elapsed:.2f}")
         return self.acc_df
 
-
     def s1_clust_by_message_length(self):
         logger.debug ("s1 start")
         #STEP1 - Cluster (aggragete) logs based on word length
@@ -226,7 +226,7 @@ def s1_clust_by_message_length(self):
             df_part = df_aggre_s1[i].with_columns(pl.col("events", "row_nr")).explode("events", "row_nr")
             df_part = df_part.with_columns(pl.col("events").list.to_struct()).unnest("events")
             df_part = df_part.drop("e_words_len", "part_len")
-            self.add_partition(Partition(df = df_part, len=len_words, split_trace="S1 "))
+            self.add_partition(_Partition(df = df_part, len=len_words, split_trace="S1 "))
         return df_aggre_s1 #For easier debugging
     
     def add_partition(self, partition, parent_partition = None):
@@ -256,7 +256,7 @@ def s2_clust_by_token_pos(self, partition):
             for key, dataframe in row_dict.items():
                 logger.debug(f"s2 least frequent for word dataframe Appending: {key}")
                 #partition.subpartitions.append(Partition(dataframe, len = partition.len,  split_trace=partition.split_trace+" S2"))
-                self.add_partition(Partition(dataframe, len = partition.len,  split_trace=partition.split_trace+"S2 "), parent_partition=partition)
+                self.add_partition(_Partition(dataframe, len = partition.len, split_trace=partition.split_trace + "S2 "), parent_partition=partition)
             partition.df = None #To save memory
         else: 
             self.df_s2_dict = None
@@ -446,12 +446,12 @@ def s3_clust_by_bijection (self, partition):
         for key, dataframe in p1_part_dict.items():
             logger.debug(f"S3P1 dict with key Appending: {key}")
             #partition.subpartitions.append(Partition(dataframe, len = partition.len, split_trace=partition.split_trace+"S3"))
-            self.add_partition(Partition(dataframe, len = partition.len,  split_trace=partition.split_trace+"S3 "), parent_partition=partition)
+            self.add_partition(_Partition(dataframe, len = partition.len, split_trace=partition.split_trace + "S3 "), parent_partition=partition)
        
         for key, dataframe in p2_part_dict.items():
             logger.debug(f"S3P2 dict with key Appending: {key}")
             #partition.subpartitions.append(Partition(dataframe, len = partition.len, split_trace=partition.split_trace+"S3"))
-            self.add_partition(Partition(dataframe, len = partition.len,  split_trace=partition.split_trace+"S3 "), parent_partition=partition)
+            self.add_partition(_Partition(dataframe, len = partition.len, split_trace=partition.split_trace + "S3 "), parent_partition=partition)
         #part_df is reduced in the process. If we do assign here we the same log rows in multiple levels
         
      
@@ -460,7 +460,6 @@ def s3_clust_by_bijection (self, partition):
         else:
             partition.df = part_df
 
-
     def _get_rank_position(self, len, card, one_to_m):
         dist = card / len
         if dist < self.lower_bound:
@@ -474,8 +473,6 @@ def _get_rank_position(self, len, card, one_to_m):
             else:
                 return 2
 
-              
-        
     def _relation_type(self, df_part, p1, p2):
         # Get the column names from indices
         col_p1 = df_part.columns[p1]
diff --git a/parsers/pyspell/LICENSE b/loglead/parsers/pyspell/LICENSE
similarity index 100%
rename from parsers/pyspell/LICENSE
rename to loglead/parsers/pyspell/LICENSE
diff --git a/parsers/pyspell/README.md b/loglead/parsers/pyspell/README.md
similarity index 100%
rename from parsers/pyspell/README.md
rename to loglead/parsers/pyspell/README.md
diff --git a/parsers/pyspell/spell.py b/loglead/parsers/pyspell/spell.py
similarity index 90%
rename from parsers/pyspell/spell.py
rename to loglead/parsers/pyspell/spell.py
index 10d88a4..f5e4f50 100644
--- a/parsers/pyspell/spell.py
+++ b/loglead/parsers/pyspell/spell.py
@@ -3,9 +3,11 @@
 
 import re
 import json
-import pickle
 
-class lcsobj():
+__all__ = ['SpellParser']
+
+
+class _lcsobj():
 
     def __init__(self, objid, seq, lineid, refmt):
         self._refmt = refmt
@@ -169,7 +171,8 @@ def _get_pos(self):
     def get_id(self):
         return self._id
 
-class lcsmap():
+
+class SpellParser():
 
     def __init__(self, refmt):
         self._refmt = refmt
@@ -183,7 +186,7 @@ def insert(self, entry):
         obj = self.match(seq)
         if obj == None:
             self._lineid += 1
-            obj = lcsobj(self._id, seq, self._lineid, self._refmt)
+            obj = _lcsobj(self._id, seq, self._lineid, self._refmt)
             self._lcsobjs.append(obj)
             self._id += 1
         else:
@@ -219,22 +222,3 @@ def dump(self):
         for i in self._lcsobjs:
             print(count, i.tojson())
             count += 1
-
-
-def save(filename, spell_lcsmap):
-    if type(spell_lcsmap) == lcsmap:
-        with open(filename,'wb') as f:
-            pickle.dump(spell_lcsmap, f)
-    else:
-        if __debug__ == True:
-            print("%s isnt slm object"%filename)
-
-def load(filename):
-    with open(filename,'rb') as f:
-        slm = pickle.load(f)
-        if type(slm) == lcsmap:
-            return slm
-        else:
-            if __debug__ == True:
-                print("%s isnt slm object"%filename)
-            return None
diff --git a/parsers/Brain/__init__.py b/parsers/Brain/__init__.py
deleted file mode 100644
index ff46cd7..0000000
--- a/parsers/Brain/__init__.py
+++ /dev/null
@@ -1,1 +0,0 @@
-from .Brain import *
diff --git a/parsers/__init__.py b/parsers/__init__.py
deleted file mode 100644
index e69de29..0000000
diff --git a/parsers/iplom/__init__.py b/parsers/iplom/__init__.py
deleted file mode 100644
index ce7c3f9..0000000
--- a/parsers/iplom/__init__.py
+++ /dev/null
@@ -1,1 +0,0 @@
-from .IPLoM import *
diff --git a/parsers/lenma/template.py b/parsers/lenma/template.py
deleted file mode 100644
index d3175be..0000000
--- a/parsers/lenma/template.py
+++ /dev/null
@@ -1,130 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-class Template(object):
-    def __init__(self, index, words, logid):
-        self._index = index
-        self._words = words
-        self._nwords = len(words)
-        self._counts = 1
-        self._logid = [logid]
-
-    @property
-    def index(self):
-        return self._index
-
-    @property
-    def words(self):
-        return self._words
-
-    @property
-    def nwords(self):
-        return self._nwords
-
-    @property
-    def counts(self):
-        return self._counts
-
-    def _dump_as_json(self):
-        """Dumps the data structure as a JSON format to serialize the
-        object.
-
-        This internal function is called by the TemplateManager
-        class.
-        """
-        assert (False)
-
-    def _restore_from_json(self, data):
-        """Initializes the instance with the provided JSON data.
-
-        This internal function is normally called by the initializer.
-        """
-        assert (False)
-
-    def get_similarity_score(self, new_words):
-        """Retruens a similarity score.
-
-        Args:
-          new_words: An array of words.
-
-        Returns:
-          score: in float.
-        """
-        assert (False)
-
-    def update(self, new_words):
-        """Updates the template data using the supplied new_words.
-        """
-        assert (False)
-
-    def __str__(self):
-        template = ' '.join([self.words[idx] if self.words[idx] != '' else '*' for idx in range(self.nwords)])
-        return '{index}({nwords})({counts}):{template}'.format(
-            index=self.index,
-            nwords=self.nwords,
-            counts=self._counts,
-            template=' '.join([self.words[idx] if self.words[idx] != '' else '*' for idx in range(self.nwords)]))
-
-
-class TemplateManager(object):
-    def __init__(self):
-        self._templates = []
-
-    @property
-    def templates(self):
-        return self._templates
-
-    def infer_template(self, words):
-        """Infer the best matching template, or create a new template if there
-        is no similar template exists.
-
-        Args:
-          words: An array of words.
-
-        Returns:
-          A template instance.
-
-        """
-        assert (False)
-
-    def dump_template(self, index):
-        """Dumps a specified template data structure usually in a text
-        format.
-
-        Args:
-          index: a template index.
-
-        Returns:
-          A serialized text data of the specified template.
-        """
-        assert (False)
-
-    def restore_template(self, data):
-        """Creates a template instance from data (usually a serialized
-        data when LogDatabase.close() method is called.
-
-        This function is called by the LogDatabase class.
-
-        Args:
-          data: a data required to rebuild a template instance.
-
-        Returns:
-          A template instance.
-        """
-        assert (False)
-
-    def _append_template(self, template):
-        """Append a template.
-
-        This internal function may be called by the LogDatabase
-        class too.
-
-        Args:
-          template: a new template to be appended.
-
-        Returns:
-          template: the appended template.
-        """
-        assert (template.index == len(self.templates))
-        self.templates.append(template)
-        return template


Test patch:
diff --git a/tests/anomaly_detectors.py b/tests/anomaly_detectors.py
index e79fb55..c34698f 100644
--- a/tests/anomaly_detectors.py
+++ b/tests/anomaly_detectors.py
@@ -1,12 +1,13 @@
-import polars as pl
 import sys
-import polars as pl
 import glob
 import os
+
+import polars as pl
+
 script_dir = os.path.dirname(os.path.abspath(__file__))
 os.chdir(script_dir)
 sys.path.append('..')
-import loglead.anomaly_detection as ad
+from loglead import AnomalyDetector
 
 # Set your directory
 home_directory = os.path.expanduser('~')
@@ -15,8 +16,8 @@
 # Get all .parquet files in the directory
 all_files = glob.glob(os.path.join(test_data_path, "*.parquet"))
 
-print ("Anomaly detectors test starting.")
-print (f"Found files {all_files}")
+print("Anomaly detectors test starting.")
+print(f"Found files {all_files}")
 # Extract unique dataset names with '_eh_' , excluding '_seq' files
 datasets = set()
 for f in all_files:
@@ -44,7 +45,7 @@
         df["normal"].sum() > 10 and
         df["anomaly"].sum() > 10):
             print(f"Running event anomaly detectors with {col}")
-            sad =  ad.AnomalyDetection(item_list_col=col, print_scores= False, store_scores=True)
+            sad = AnomalyDetector(item_list_col=col, print_scores= False, store_scores=True)
             sad.test_train_split (df, test_frac=0.5) 
             sad.evaluate_all_ads(disabled_methods=disabled_methods)
 
@@ -64,7 +65,7 @@
             df["normal"].sum() > 10 and
             df["anomaly"].sum() > 10):
                 print(f"Running seqeuence anomaly detectors with {col}")
-                sad =  ad.AnomalyDetection(item_list_col=col, print_scores= False, store_scores=True)
+                sad =  AnomalyDetector(item_list_col=col, print_scores= False, store_scores=True)
                 #High training fraction to ensure we always have suffiecient samples as these are reduced dataframes 
                 sad.test_train_split (df_seq, test_frac=0.2) 
                 sad.evaluate_all_ads(disabled_methods=disabled_methods)
@@ -73,9 +74,9 @@
         "anomaly" in df_seq.columns and 
         df_seq["normal"].sum() > 10 and
         df_seq["anomaly"].sum() > 10):
-            sad =  ad.AnomalyDetection(numeric_cols = numeric_cols, print_scores= False, store_scores=True)
+            sad = AnomalyDetector(numeric_cols = numeric_cols, print_scores= False, store_scores=True)
             #High training fraction to ensure we always have suffiecient samples as these are reduced dataframes 
             sad.test_train_split (df_seq, test_frac=0.2) 
             sad.evaluate_all_ads(disabled_methods=[])        
 
-print ("Anomaly detectors test complete.")
\ No newline at end of file
+print("Anomaly detectors test complete.")
\ No newline at end of file
diff --git a/tests/enhancers.py b/tests/enhancers.py
index 82209e4..c864a88 100644
--- a/tests/enhancers.py
+++ b/tests/enhancers.py
@@ -1,26 +1,28 @@
-import polars as pl
 import sys
-import polars as pl
 import glob
 import os
+
+import polars as pl
+
 script_dir = os.path.dirname(os.path.abspath(__file__))
 os.chdir(script_dir)
 sys.path.append('..')
-import loglead.enhancer as eh
-import loglead.loaders.base as load
+
+from loglead.loaders import BaseLoader
+from loglead.enhancers import EventLogEnhancer, SequenceEnhancer
 
 home_directory = os.path.expanduser('~')
 test_data_path = os.path.join(home_directory, "Datasets", "test_data") 
 
 # Get all .parquet files in the directory
 all_files = glob.glob(os.path.join(test_data_path, "*.parquet"))
-print ("Enhancers test starting.")
-print (f"Found files {all_files}")
+print("Enhancers test starting.")
+print(f"Found files {all_files}")
 # Extract unique dataset names, excluding '_seq' files
 datasets = set()
 for f in all_files:
     basename = os.path.basename(f)
-    #Only use main files (no _seq) that exist after loader (no _eh)
+    # Only use main files (no _seq) that exist after loader (no _eh)
     if "_lo" in basename and "_seq" not in basename and "_eh" not in basename:
         dataset_name = basename.replace(".parquet", "")
         datasets.add(dataset_name)
@@ -31,71 +33,71 @@
     primary_file = os.path.join(test_data_path, f"{dataset}.parquet")
     print(f"\nLoading {primary_file}")
     df = pl.read_parquet(primary_file)
-    #Kill nulls if they still exist
+    # Kill nulls if they still exist
     df = df.filter(pl.col("m_message").is_not_null())
     # Enhance the event data
-    print ("Enhancing data:", end=": ")
-    enhancer = eh.EventLogEnhancer(df)
-    print ("event lengths",  end=", ")
+    print("Enhancing data:", end=": ")
+    enhancer = EventLogEnhancer(df)
+    print("event lengths",  end=", ")
     df = enhancer.length()
-    print ("normalizing",   end=", ")
+    print("normalizing",   end=", ")
     df = enhancer.normalize()
-    print ("spliting to words",   end=", ")
+    print("splitting to words",   end=", ")
     df = enhancer.words()
-    print ("spliting to alphanumerics",   end=", ")
+    print("splitting to alphanumerics",   end=", ")
     df = enhancer.alphanumerics()
-    print ("spliting to trigrams",   end=", ")
+    print("splitting to trigrams",   end=", ")
     df = enhancer.trigrams()
-    print ("Drain parsing",   end=", ")
+    print("Drain parsing",   end=", ")
     df = enhancer.parse_drain()
-    print ("Tipping parsing",   end=", ")
+    print("Tipping parsing",   end=", ")
     df = enhancer.parse_tip()
 
     # Enhance / Aggregate sequence level
-    loader = load.BaseLoader(filename=None, df=None, df_seq = None)
+    loader = BaseLoader(filename=None, df=None, df_seq=None)
     seq_file = primary_file.replace(f"{dataset}.parquet", f"{dataset}_seq.parquet")
     if os.path.exists(seq_file):
         df_seq = pl.read_parquet(seq_file)
         loader.df_seq = df_seq
-        enhancer_seq = eh.SequenceEnhancer(df = df, df_seq = df_seq)
-        print ("\nAggregating drain parsing results",   end=", ")
+        enhancer_seq = SequenceEnhancer(df=df, df_seq=df_seq)
+        print("\nAggregating drain parsing results",   end=", ")
         df_seq = enhancer_seq.events()
-        print ("\nCreating next-event-prediction results from Drain events",   end=", ")
+        print("\nCreating next-event-prediction results from Drain events",   end=", ")
         df_seq = enhancer_seq.next_event_prediction()
-        print ("Aggregating tokens / words",   end=", ")
+        print("Aggregating tokens / words",   end=", ")
         df_seq = enhancer_seq.tokens()
         df_seq = enhancer_seq.tokens("e_trigrams")
-        print ("Aggregating event lengths",   end=", ") 
+        print("Aggregating event lengths",   end=", ")
         df_seq = enhancer_seq.eve_len()
-        print ("Enhancing sequence duration",   end=", ")
+        print("Enhancing sequence duration",   end=", ")
         df_seq = enhancer_seq.start_time()
         df_seq = enhancer_seq.end_time()
         df_seq = enhancer_seq.duration()
-        print ("Enhancing sequence length in events")
+        print("Enhancing sequence length in events")
         df_seq = enhancer_seq.seq_len()
-        #Preparing loader for addition reduction
+        # Preparing loader for addition reduction
         loader.df_seq = df_seq
     loader.df = df
-    #Save the data used for anomaly_detectors tests. 
+    # Save the data used for anomaly_detectors tests.
     loader.df.write_parquet(f"{test_data_path}/{dataset}_eh.parquet") 
     if os.path.exists(seq_file):
         loader.df_seq.write_parquet(f"{test_data_path}/{dataset}_eh_seq.parquet")
-    #Test the remaining slow parsers.   
-    df = loader.reduce_dataframes(frac=0.01)#Reducing the ~100k -> 1k
+    # Test the remaining slow parsers.
+    df = loader.reduce_dataframes(frac=0.01)  # Reducing the ~100k -> 1k
     df_seq = loader.df_seq
     enhancer.df = df
     print(f"Remaining parsers with lines {len(df)}", end=": ")
-    print ("IpLom",   end=", ")
+    print("IpLom",   end=", ")
     df = enhancer.parse_iplom()
-    print ("Pl-Iplom parsing",   end=", ")
+    print("Pl-Iplom parsing",   end=", ")
     df = enhancer.parse_pliplom()
-    print ("AEL parsing",   end=", ")
+    print("AEL parsing",   end=", ")
     df = enhancer.parse_ael()
-    print ("Brain parsing",   end=", ")
+    print("Brain parsing",   end=", ")
     df = enhancer.parse_brain()
-    print ("Spell parsing",   end=", ")
+    print("Spell parsing",   end=", ")
     df = enhancer.parse_spell()
-    print ("LenMa parsing",  end=", ")
+    print("LenMa parsing",  end=", ")
     df = enhancer.parse_lenma()
     try:
         import tensorflow as tf
@@ -107,5 +109,4 @@
         print("\nTensorFlow is not installed. Embedding creation not tested")
         # Add any alternative code here if TensorFlow is not available
 
-
-print ("Enhancers test complete.")
\ No newline at end of file
+print("Enhancers test complete.")
diff --git a/tests/loaders.py b/tests/loaders.py
index 91160bc..f7c9f67 100644
--- a/tests/loaders.py
+++ b/tests/loaders.py
@@ -1,20 +1,22 @@
 
  
-#Separate demo files
+# Separate demo files
 import sys
 
-import polars as pl
 import psutil
 import os
+
+
 script_dir = os.path.dirname(os.path.abspath(__file__))
 os.chdir(script_dir)
 sys.path.append('..')
-import loglead.loaders.base as load
-import loglead.loaders.supercomputers as load_sc
-import loglead.loaders.hdfs as load_hdfs
-import loglead.loaders.hadoop as load_hadoop
-import loglead.loaders.pro as load_pro
-import loglead.loaders.nezha as load_nezha
+
+from loglead.loaders import BGLLoader
+from loglead.loaders import ThuSpiLibLoader
+from loglead.loaders import HDFSLoader
+from loglead.loaders import HadoopLoader
+from loglead.loaders import ProLoader
+from loglead.loaders import NezhaLoader
 
 home_directory = os.path.expanduser('~')
 full_data = os.path.join(home_directory, "Datasets")
@@ -23,7 +25,7 @@
 memory_limit_NEZHA_WS = 30
 
 
-print ("Loaders test starting.")
+print("Loaders test starting.")
 # Dictionary to store file paths for each dataset
 # If you are missing any datasets. Comment them out. 
 data_files = {
@@ -35,7 +37,7 @@
     "bgl": {
         "log_file": f"{full_data}/bgl/BGL.log"
     },
-    "pro":{
+    "pro": {
         "log_file": f"{full_data}/profilence/*.txt"
     },
     "hdfs": {
@@ -53,68 +55,70 @@
         "log_file": f"{full_data}/liberty/liberty2.log",
     },
     "nezha_tt": {
-        "log_file":f"{full_data}/nezha/",
+        "log_file": f"{full_data}/nezha/",
         "system": "TrainTicket"
         },
-   "nezha_ws": {
-        "log_file":f"{full_data}/nezha/",
+    "nezha_ws": {
+        "log_file": f"{full_data}/nezha/",
         "system": "WebShop"
         }
     
 }
 
+
 def create_correct_loader(dataset):
     loader = None
     if dataset == "hdfs":
-        loader = load_hdfs.HDFSLoader(filename=data_files["hdfs"]["log_file"], 
-                                 labels_file_name=data_files["hdfs"]["labels_file"])
-    elif dataset == "tb": #Must have gbs for TB
+        loader = HDFSLoader(filename=data_files["hdfs"]["log_file"],
+                            labels_file_name=data_files["hdfs"]["labels_file"])
+    elif dataset == "tb":  # Must have gbs for TB
         if memory > memory_limit_TB:
             # Might take 4-6 minutes in HPC VM. In desktop out of memory
-            loader = load_sc.ThuSpiLibLoader(filename=data_files["tb"]["log_file"]) 
+            loader = ThuSpiLibLoader(filename=data_files["tb"]["log_file"])
         else:
-            loader = load_sc.ThuSpiLibLoader(filename=data_files["tb"]["log_file_2k"]) 
+            loader = ThuSpiLibLoader(filename=data_files["tb"]["log_file_2k"])
     elif dataset == "spirit": 
         if memory > memory_limit_TB:
             # Might take 4-6 minutes in HPC VM. In desktop out of memory
-            loader = load_sc.ThuSpiLibLoader(filename=data_files["spirit"]["log_file"]) 
+            loader = ThuSpiLibLoader(filename=data_files["spirit"]["log_file"])
         else:
             print("Skipping spirit due to memory limit") 
     elif dataset == "liberty": 
         if memory > memory_limit_TB:
             # Might take 4-6 minutes in HPC VM. In desktop out of memory
-            loader = load_sc.ThuSpiLibLoader(filename=data_files["liberty"]["log_file"], split_component=False) 
+            loader = ThuSpiLibLoader(filename=data_files["liberty"]["log_file"], split_component=False)
         else:
             print("Skipping liberty due to memory limit") 
     elif dataset == "bgl":
-        loader = load_sc.BGLLoader(filename=data_files["bgl"]["log_file"])
-    elif dataset=="pro":
-       loader = load_pro.ProLoader(filename=data_files["pro"]["log_file"])
+        loader = BGLLoader(filename=data_files["bgl"]["log_file"])
+    elif dataset == "pro":
+        loader = ProLoader(filename=data_files["pro"]["log_file"])
     elif dataset == "hadoop":
-        loader = load_hadoop.HadoopLoader(filename=data_files["hadoop"]["log_dir"],
-                                   filename_pattern=data_files["hadoop"]["filename_pattern"],
-                                   labels_file_name=data_files["hadoop"]["labels_file"]) 
+        loader = HadoopLoader(filename=data_files["hadoop"]["log_dir"],
+                              filename_pattern=data_files["hadoop"]["filename_pattern"],
+                              labels_file_name=data_files["hadoop"]["labels_file"])
     elif dataset == "nezha_tt":
-        loader = load_nezha.NezhaLoader(filename= data_files["nezha_tt"]["log_file"],
-                                        system = data_files["nezha_tt"]["system"]) 
+        loader = NezhaLoader(filename=data_files["nezha_tt"]["log_file"],
+                             system=data_files["nezha_tt"]["system"])
     elif dataset == "nezha_ws":
         if memory > memory_limit_NEZHA_WS:
-            loader = load_nezha.NezhaLoader(filename= data_files["nezha_ws"]["log_file"],
-                                            system = data_files["nezha_ws"]["system"])
+            loader = NezhaLoader(filename=data_files["nezha_ws"]["log_file"],
+                                 system=data_files["nezha_ws"]["system"])
         else:
             print("Skipping liberty due to nezha_ws limit")      
     return loader
 
+
 def check_and_save(dataset, loader):
-    #Create a test datafolder
+    # Create a test datafolder
     test_data_path = os.path.join(full_data, "test_data")
     os.makedirs(test_data_path, exist_ok=True)
     if dataset == "hdfs":
         if len(loader.df) != 11175629:
             print(f"MISMATCH! hdfs expected 11175629 was {len(loader.df)}")
-        loader.reduce_dataframes(frac=0.01) #Reduce all to roughly 100k rows
+        loader.reduce_dataframes(frac=0.01)  # Reduce all to roughly 100k rows
         
-    elif dataset == "tb": #Must have gbs for TB
+    elif dataset == "tb":  # Must have gbs for TB
         if memory > memory_limit_TB:
             if len(loader.df) != 211212192:
                 print(f"MISMATCH tb expected 211212192 was {len(loader.df)}")
@@ -122,32 +126,32 @@ def check_and_save(dataset, loader):
         else:
             if len(loader.df) != 2000:
                 print(f"MISMATCH tb expected 2000 was {len(loader.df)}")
-    elif dataset == "spirit" or dataset == "liberty": #Must have gbs for TB
+    elif dataset == "spirit" or dataset == "liberty":  # Must have gbs for TB
         loader.reduce_dataframes(frac=0.0005)
     elif dataset == "bgl":
         if len(loader.df) != 4747963:
             print(f"MISMATCH bgl expected 4747963 was {len(loader.df)}")
-        loader.reduce_dataframes(frac=0.02) #Reduce all to roughly 100k rows
+        loader.reduce_dataframes(frac=0.02)  # Reduce all to roughly 100k rows
     elif dataset == "pro":
         if len(loader.df) != 5203599:
             print(f"MISMATCH pro expected 5203599 was {len(loader.df)}")
-        loader.reduce_dataframes(frac=0.02) #Reduce all to roughly 100k rows
+        loader.reduce_dataframes(frac=0.02)  # Reduce all to roughly 100k rows
     elif dataset == "hadoop":
         if len(loader.df) != 177592:
             print(f"MISMATCH hadoop expected 177592 was {len(loader.df)}")
-        loader.reduce_dataframes(frac=0.56) #Reduce all to roughly 100k rows
+        loader.reduce_dataframes(frac=0.56)  # Reduce all to roughly 100k rows
     elif dataset == "nezha_tt":
         if len(loader.df) != 272270:
             print(f"MISMATCH nezha_tt expected 272270 was {len(loader.df)}")
-        loader.reduce_dataframes(frac=0.33) #Reduce all to roughly 100k rows
+        loader.reduce_dataframes(frac=0.33)  # Reduce all to roughly 100k rows
     elif dataset == "nezha_ws":
         if len(loader.df) != 3958203:
             print(f"MISMATCH nezha_ws expected 3 958 203 was {len(loader.df)}")
-        loader.reduce_dataframes(frac=0.025) #Reduce all to roughly 100k rows
+        loader.reduce_dataframes(frac=0.025)  # Reduce all to roughly 100k rows
     else:
         print(f"Invalid dataset {dataset}")
         return
-    #Save the data used in enhancer tests. 
+    # Save the data used in enhancer tests.
     loader.df.write_parquet(f"{test_data_path}/{dataset}_lo.parquet") 
     if any(sub in dataset for sub in ["hdfs", "pro", "hadoop"]):
         loader.df_seq.write_parquet(f"{test_data_path}/{dataset}_lo_seq.parquet")  
@@ -157,9 +161,9 @@ def check_and_save(dataset, loader):
     memory = psutil.virtual_memory().available / (1024 ** 3)
     print(f"Loading: {key}")
     loader = create_correct_loader(key)
-    if (loader == None):
+    if loader is None:
         continue
     loader.execute()
     check_and_save(key, loader)
 
-print ("Loading test complete.")
\ No newline at end of file
+print("Loading test complete.")