Repository: spacetelescope/jwql

Issue:
Determine how best to store monitoring data
Now that we have a few monitors in the works (#229 , #269), we should decide on a generalized/standard way to store monitoring data.  This could take the form of a generalized class or module with convenience functions for saving, reading, displaying data.


Discussion:
Just to get the conversation started, here are dark monitor-related quantities that I envision possibly storing:

1. The date and time of the last MAST query for each instrument/detector/aperture where the dark monitor ran (as opposed to quitting because there was not enough new data). This will be used as the starting time of the next query.
2. Newly identified hot/dead/noisy pixel coordinates, for possible comparison against the current reference file, and for visualizing on the web app.
3. Running history of mean and stdev of the dark within each instrument/detector/aperture and possibly within each amp, in order to create trending plots.
4. Histogram and best-fit Gaussian parameters. Keep these only long enough to make a plot of the current run? Keep longer term so that histograms and Gaussian fit parameters can be tracked over time?

So far all of these things are lists of numbers, and so they shouldn't be too difficult to store. I wonder if any of the monitors will want to store images. I suppose in that case there can be a table of some sort that connects fits/png/etc filenames to a given run of a monitor.
Actually there are a couple images that would be worth storing for the dark monitor: The "baseline" or comparison mean dark slope image and stdev image. These would be needed for each instrument/detector/aperture combination.

Resolving patch:
diff --git a/jwql/database/database_interface.py b/jwql/database/database_interface.py
--- a/jwql/database/database_interface.py
+++ b/jwql/database/database_interface.py
@@ -24,6 +24,8 @@
     - Joe Filippazzo
     - Johannes Sahlmann
     - Matthew Bourque
+    - Lauren Chambers
+    - Bryan Hilbert
 
 Use
 ---
@@ -54,16 +56,22 @@
 """
 
 from datetime import datetime
+import os
 import socket
 
 import pandas as pd
 from sqlalchemy import Boolean
 from sqlalchemy import Column
 from sqlalchemy import create_engine
+from sqlalchemy import Date
 from sqlalchemy import DateTime
+from sqlalchemy import Enum
+from sqlalchemy import Float
 from sqlalchemy import Integer
 from sqlalchemy import MetaData
 from sqlalchemy import String
+from sqlalchemy import Time
+from sqlalchemy import UniqueConstraint
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.orm import sessionmaker
 from sqlalchemy.orm.query import Query
@@ -188,6 +196,132 @@ def colnames(self):
         return a_list
 
 
+class Monitor(base):
+    """ORM for the ``monitor`` table"""
+
+    # Name the table
+    __tablename__ = 'monitor'
+
+    id = Column(Integer, primary_key=True)
+    monitor_name = Column(String(), nullable=False)
+    start_time = Column(DateTime, nullable=False)
+    end_time = Column(DateTime, nullable=True)
+    status = Column(Enum('SUCESS', 'FAILURE'), nullable=True)
+    affected_tables = Column(Array, nullable=True)
+    log_file(Column(String(), nullable=False))
+
+
+def get_monitor_columns(data_dict, table_name):
+    """Read in the corresponding table definition text file to
+    generate ``SQLAlchemy`` columns for the table.
+
+    Parameters
+    ----------
+    data_dict : dict
+        A dictionary whose keys are column names and whose values
+        are column definitions.
+    table_name : str
+        The name of the database table
+
+    Returns
+    -------
+    data_dict : dict
+        An updated ``data_dict`` with the approriate columns for
+        the monitor added.
+    """
+
+    # Define column types
+    data_type_dict = {'integer': Integer(),
+                      'string': String(),
+                      'float': Float(precision=32),
+                      'decimal': Float(precision='13,8'),
+                      'date': Date(),
+                      'time': Time(),
+                      'datetime': DateTime,
+                      'bool': Boolean}
+
+    # Get the data from the table definition file
+    table_definition_file = os.path.join(os.path.split(__file__)[0],
+                                         'monitor_table_definitions',
+                                         '{}.txt'.format(table_name))
+    with open(table_definition_file, 'r') as f:
+        data = f.readlines()
+
+    # Parse out the column names from the data types
+    column_definitions = [item.strip().split(', ') for item in data]
+    for column_definition in column_definitions:
+        column_name = column_definition[0]
+        data_type = column_definition[1]
+
+        # Create a new column
+        if data_type in list(data_type_dict.keys()):
+            data_dict[column_name.lower()] = Column(data_type_dict[data_type])
+        else:
+            raise ValueError('Unrecognized column type: {}:{}'.format(column_name, data_type))
+
+    return data_dict
+
+
+def get_monitor_table_constraints(data_dict, table_name):
+    """Add any necessary table constrains to the given table via the
+    ``data_dict``.
+
+    Parameters
+    ----------
+    data_dict : dict
+        A dictionary whose keys are column names and whose values
+        are column definitions.
+    table_name : str
+        The name of the database table
+
+    Returns
+    -------
+    data_dict : dict
+        An updated ``data_dict`` with the approriate table constraints
+        for the monitor added.
+    """
+
+    return data_dict
+
+
+def monitor_orm_factory(class_name):
+    """Create a ``SQLAlchemy`` ORM Class for a ``jwql`` instrument
+    monitor.
+
+    Parameters
+    ----------
+    class_name : str
+        The name of the class to be created
+
+    Returns
+    -------
+    class : obj
+        The ``SQLAlchemy`` ORM
+    """
+
+    # Initialize a dictionary to hold the column metadata
+    data_dict = {}
+    data_dict['__tablename__'] = class_name.lower()
+
+    # Columns specific to all monitor ORMs
+    data_dict['id'] = Column(Integer, primary_key=True, nullable=False)
+    data_dict['entry_date'] = Column(DateTime, unique=True, nullable=False, default=datetime.now())
+    data_dict['__table_args__'] = (UniqueConstraint('id', 'entry_date', name='monitor_uc'),)
+
+    # Get monitor-specific columns
+    data_dict = get_monitor_columns(data_dict, data_dict['__tablename__'])
+
+    # Get monitor-specific table constrains
+    data_dict = get_monitor_table_constraints(data_dict, data_dict['__tablename__'])
+
+    return type(class_name, (base,), data_dict)
+
+# Create tables from ORM factory
+# NIRCamDarkQueries = monitor_orm_factory('nircam_dark_queries')
+# NIRCamDarkPixelStats = monitor_orm_factory('nircam_dark_pixel_stats')
+# NIRCamDarkDarkCurrent = monitor_orm_factory('nircam_dark_dark_current')
+
+
 if __name__ == '__main__':
 
     base.metadata.create_all(engine)


Test patch: