Upload 2 files

App_Function_Libraries/Metrics/logger_config.py

@@ -0,0 +1,58 @@
+# logger_config.py
+#
+# Imports
+import logging
+from logging.handlers import RotatingFileHandler
+from pythonjsonlogger import jsonlogger
+import os
+#
+############################################################################################################
+#
+# Functions:
+
+def setup_logger(log_file_path="tldw_app_logs.json"):
+    """
+    Sets up the logger with both StreamHandler and FileHandler, formatted in JSON.
+
+    Parameters:
+        log_file_path (str): Path to the JSON log file.
+
+    Returns:
+        logging.Logger: Configured logger instance.
+    """
+    logger = logging.getLogger("tldw_app_logs")
+    logger.setLevel(logging.DEBUG)  # Set to DEBUG for detailed logs
+
+    # Prevent adding multiple handlers if the logger is already configured
+    if not logger.handlers:
+        # StreamHandler for console output
+        stream_handler = logging.StreamHandler()
+        stream_formatter = jsonlogger.JsonFormatter(
+            '%(asctime)s %(levelname)s %(name)s event %(event)s type %(type)s value %(value)s labels %(labels)s timestamp %(timestamp)s'
+        )
+        stream_handler.setFormatter(stream_formatter)
+        logger.addHandler(stream_handler)
+
+        # Ensure the directory for log_file_path exists
+        log_dir = os.path.dirname(log_file_path)
+        if log_dir and not os.path.exists(log_dir):
+            os.makedirs(log_dir, exist_ok=True)
+
+        # RotatingFileHandler for writing logs to a JSON file with rotation
+        file_handler = RotatingFileHandler(
+            log_file_path, maxBytes=10*1024*1024, backupCount=5  # 10 MB per file, keep 5 backups
+        )
+        file_formatter = jsonlogger.JsonFormatter(
+            '%(asctime)s %(levelname)s %(name)s event %(event)s type %(type)s value %(value)s labels %(labels)s timestamp %(timestamp)s'
+        )
+        file_handler.setFormatter(file_formatter)
+        logger.addHandler(file_handler)
+
+    return logger
+
+# Initialize the logger
+logger = setup_logger()
+
+#
+# End of Functions
+############################################################################################################

App_Function_Libraries/Metrics/metrics_logger.py

@@ -0,0 +1,98 @@
+# metrics_logger.py
+#
+# Imports
+from datetime import datetime, timezone
+#
+# Third-party Imports
+#
+# Local Imports
+from App_Function_Libraries.Metrics.logger_config import logger
+#
+############################################################################################################
+#
+# Functions:
+
+def log_counter(metric_name, labels=None, value=1):
+    log_entry = {
+        "event": metric_name,
+        "type": "counter",
+        "value": value,
+        "labels": labels or {},
+        # datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC).
+        # FIXME
+        "timestamp": datetime.now(timezone.utc).isoformat() + "Z"
+    }
+    logger.info("metric", extra=log_entry)
+
+def log_histogram(metric_name, value, labels=None):
+    log_entry = {
+        "event": metric_name,
+        "type": "histogram",
+        "value": value,
+        "labels": labels or {},
+        # datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC).
+        # FIXME
+        "timestamp": datetime.now(timezone.utc).isoformat() + "Z"
+    }
+    logger.info("metric", extra=log_entry)
+
+#
+# End of Functions
+############################################################################################################
+
+# # Prometheus
+# # metrics_logger.py (Prometheus version)
+# from prometheus_client import Counter, Histogram, start_http_server
+# import logging
+# from functools import wraps
+# import time
+#
+# # Initialize Prometheus metrics
+# VIDEOS_PROCESSED = Counter('videos_processed_total', 'Total number of videos processed', ['whisper_model', 'api_name'])
+# VIDEOS_FAILED = Counter('videos_failed_total', 'Total number of videos failed to process', ['whisper_model', 'api_name'])
+# TRANSCRIPTIONS_GENERATED = Counter('transcriptions_generated_total', 'Total number of transcriptions generated', ['whisper_model'])
+# SUMMARIES_GENERATED = Counter('summaries_generated_total', 'Total number of summaries generated', ['whisper_model'])
+# VIDEO_PROCESSING_TIME = Histogram('video_processing_time_seconds', 'Time spent processing videos', ['whisper_model', 'api_name'])
+# TOTAL_PROCESSING_TIME = Histogram('total_processing_time_seconds', 'Total time spent processing all videos', ['whisper_model', 'api_name'])
+#
+# def init_metrics_server(port=8000):
+#     start_http_server(port)
+#
+# def log_counter(metric_name, labels=None, value=1):
+#     if metric_name == "videos_processed_total":
+#         VIDEOS_PROCESSED.labels(**(labels or {})).inc(value)
+#     elif metric_name == "videos_failed_total":
+#         VIDEOS_FAILED.labels(**(labels or {})).inc(value)
+#     elif metric_name == "transcriptions_generated_total":
+#         TRANSCRIPTIONS_GENERATED.labels(**(labels or {})).inc(value)
+#     elif metric_name == "summaries_generated_total":
+#         SUMMARIES_GENERATED.labels(**(labels or {})).inc(value)
+#
+# def log_histogram(metric_name, value, labels=None):
+#     if metric_name == "video_processing_time_seconds":
+#         VIDEO_PROCESSING_TIME.labels(**(labels or {})).observe(value)
+#     elif metric_name == "total_processing_time_seconds":
+#         TOTAL_PROCESSING_TIME.labels(**(labels or {})).observe(value)
+
+
+# # main.py or equivalent entry point
+# from metrics_logger import init_metrics_server
+#
+#
+# def main():
+#     # Start Prometheus metrics server on port 8000
+#     init_metrics_server(port=8000)
+#
+#     # Initialize and launch your Gradio app
+#     create_video_transcription_tab()
+#
+#
+# if __name__ == "__main__":
+#     main()
+
+# prometheus.yml
+# scrape_configs:
+#   - job_name: 'video_transcription_app'
+#     static_configs:
+#       - targets: ['localhost:8000']  # Replace with your application's host and port
+