adapt logmetric to also make local per-logmsg test using sacrebleu

logmetric.py

@@ -67,7 +67,7 @@ class LogMetric(evaluate.Metric):
     """TODO: Short description of my evaluation module."""
 
     # Constant regex to get timestrings
-    timestamp_regex = r'(^\d{4}[-/.]\d{2}[-/.]\d{2}(?:[ T]\d{2}[:]\d{2}(?:[:]\d{2}(?:[.,]\d+)?)?(?:Z|[+-]\d{2}[:]\d{2})?)?)'
+    timestamp_regex = r'^\s*(\d{4}[-/.]\d{2}[-/.]\d{2}(?:[ T]\d{2}[:]\d{2}(?:[:]\d{2}(?:[.,]\d+)?)?(?:Z|[+-]\d{2}[:]\d{2})?)?)\s*'
     timestamp_pattern = re.compile(timestamp_regex, re.MULTILINE)
 
     def _info(self):
@@ -96,54 +96,104 @@ class LogMetric(evaluate.Metric):
         # TODO: Download external resources if needed
         pass
 
-    def getLogMetric(self, pred : str, ref : str):
+    def getLogMetric(self, pred : str, ref : str, sacrebleu):
         ref = ref.strip(' \t\n\r')
         pred = pred.strip(' \t\n\r')
         
         # Find all timestrings in the log
-        pred_timestrings = self.timestamp_pattern.findall(pred)
-        ref_timestrings = self.timestamp_pattern.findall(ref)
-
-        #Check if there is the correct amount of timestrings in the prediction
-        if(len(pred_timestrings) != len(ref_timestrings)):
-            return 0.0
+        # pred_timestrings = self.timestamp_pattern.findall(pred)
+        pred_split_log = self.timestamp_pattern.split(pred)
+        # ref_timestrings = self.timestamp_pattern.findall(ref)
+        ref_split_log = self.timestamp_pattern.split(ref)
+
+        # This should alwas hold (safety feature)
+        # TODO: remove this after testing
+        assert(len(pred_split_log) % 2 == len(ref_split_log) % 2 == 1)
+
+        # One logentry always consists of timestamp + log-message
+        pred_logentries = []
+        ref_logentries = []
+
+        # reorganize log into logentry-tuples, consisting of timestamp + log-message
+        for i in range(1, len(pred_split_log), 2):
+            pred_logentries.append((pred_split_log[i],pred_split_log[i+1]))
         
-        # If there are no timestrings, we must not check anything, we can directly return 1.0
-        if (len(pred_timestrings) == 0):
-            return 1.0
+        for i in range(1, len(ref_split_log), 2):
+            ref_logentries.append((ref_split_log[i],ref_split_log[i+1]))
+
+        # The number of logentries of the reference/prediction which has more/less entries/timestamps
+        max_logentries = max(len(pred_logentries), len(ref_logentries))
+        min_logentries = min(len(pred_logentries), len(ref_logentries))
+                
+
+        # Case there are no timestamps in reference and none in prediction
+        # we can compute bleu directly from original prediction (ref will be empty, but we offload this to the bleu metric)
+        if (len(pred_logentries) == 0 and len(ref_logentries) == 0):
+            # TODO: remove this later, for testing purposes only
+            assert(pred == "")
+            # any sensible log reference is empty if there is no timestamp, hence it suffices to check exact match
+            logmsg_score = 100.0 if pred == ref else 0.0
+            return 0.3 * 100.0 + 0.7 * logmsg_score
         
+        # Case one has 0 timestamps, other has >0 timestamps
+        if (len(pred_logentries) == 0 or len(ref_logentries) == 0):
+            # It is nonsensical to compare something in this case
+            return 0.0
+
+ 
         # replace all digits in the reference timestamp (first timestamp) with '/d' to get
         # a regex that describes the format  
-        pred_timestring_pattern = re.sub(r'\d', r'\\d', re.escape(pred_timestrings[0]))
+        pred_timestring_pattern = re.sub(r'\d', r'\\d', re.escape(pred_logentries[0][0]))
         
+        matchesPatternScore = 100.0
+        monotonicallyIncreasingScore = 100.0
+        
+        # An array to save score per logentry
+        logmessage_scores = []
+        # TODO: Idea to penalize too long/ short logs-> add the amount of(max_len - min_len) between timestamps times score 0 at the end
         # A variable to save the previous timestamp (as datetime obj) to check monotonicity
         prev_datetime = None
         # Convert matches to datetime objects
-        for ts in pred_timestrings:
+        # TODO TODO TODO fix this:
+        for i in range(min_logentries):
+            ts, pred_lm = pred_logentries[i]
+            _, ref_lm = ref_logentries[i]
             try:
                 # Check if the format matches with the format of the first timestamp
+                # TODO!! Check this later, maybe it is too restricting for training a llm
                 matchesPattern = re.fullmatch(pred_timestring_pattern, ts) is not None
                 # Check if the timestamps are monotonically increasing
                 cur_datetime = dateutil.parser.parse(ts)
                 monotonicallyIncreasing = True if prev_datetime == None else prev_datetime <= cur_datetime
                 prev_datetime = cur_datetime
 
-                if not (matchesPattern and monotonicallyIncreasing):
-                    # timestamps not consistent
-                    return 0.0
+                # If one entry doesn't fulfill the matching pattern property or the monotinicity property, set to 0 for whole log
+                if (not matchesPattern):
+                    matchesPatternScore = 0.0
+                if (not monotonicallyIncreasing):
+                    monotonicallyIncreasingScore = 0.0
 
             except Exception as e:
                 # e.g. date format not parsable by dateutil.parser
-                return 0.0
-
+                matchesPatternScore = 0.0
+                monotonicallyIncreasingScore = 0.0
+            
+            logmessage_scores.append(sacrebleu.compute(predictions=[pred_lm], references=[ref_lm])["score"])
+
+        # TODO: remove later. Used only for testing purposes
+        assert(len(logmessage_scores) == min_logentries)
+        # we aggregate the bleu scores where we weight the difference in logentries with a score of 0
+        logmessage_aggregated_score = ((min_logentries / max_logentries) * np.mean(logmessage_scores))
         # Correct amt of timestrings, monotonically increasing, consistent + (by dateutil.parser) parsable format
-        return 1.0
+        return 0.2 * monotonicallyIncreasingScore + 0.1 * matchesPatternScore + 0.7 * logmessage_aggregated_score
 
-    def _compute(self, predictions, references):
+    def _compute(self, predictions, references, sacrebleu):
         """Returns the scores"""
 
+        # TODO: get separate log entries (split before timestamps), replace timestamps with token and compare the log entry with BLEU
+
         t_before_logmetric = time.perf_counter()
-        timestamp_score = np.mean([self.getLogMetric(p,r) for p,r in zip(predictions,references)])
+        timestamp_score = np.mean([self.getLogMetric(p,r, sacrebleu) for p,r in zip(predictions,references)])
         t_after_logmetric = time.perf_counter()
 
         logmetric_duration = f" {t_after_logmetric - t_before_logmetric:0.10f}"