Repository: debrief/pepys-import

Issue:
Minor - use (x,y) location objects
There's currently a bug whereby location is only expressed in degrees.

We should switch to passing self-describing completed `State2` objects around.

Discussion:

Patch:
diff --git a/docs/usage.rst b/docs/usage.rst
index 625b98fa..d8edb959 100644
--- a/docs/usage.rst
+++ b/docs/usage.rst
@@ -4,4 +4,16 @@ Usage
 
 To use pepys-import in a project::
 
-    import pepys_import
+    from pepys_import.file.file_processor import FileProcessor
+    from pepys_import.file.replay_parser import ReplayParser
+    from pepys_import.file.nmea_parser import NMEAParser
+
+    # create file processor, give it target filename
+    processor = FileProcessor("trial_db.db")
+
+    # register a couple of sample parsers
+    processor.register(ReplayParser())
+    processor.register(NMEAParser())
+
+    # instruct Pepys to process this directory
+    processor.process(".")
diff --git a/pepys_import/core/formats/state2.py b/pepys_import/core/formats/state2.py
index 8046376a..8eeb76df 100644
--- a/pepys_import/core/formats/state2.py
+++ b/pepys_import/core/formats/state2.py
@@ -34,17 +34,20 @@ class State2:
             )
         )
 
-    def set_speed(self, speed):
+    def set_speed(self, speed: quantity):
         self.speed = speed
 
     def set_heading(self, heading: quantity):
         self.heading = heading
 
-    def set_latitude(self):
-        pass
+    def set_location(self, lat_val: float, long_val: float):
+        self.location = (lat_val, long_val)
 
-    def set_longitude(self):
-        pass
+    def set_location_obj(self, location):
+        self.location = location
+
+    def set_longitude(self, long_val: float):
+        self.longitude = long_val
 
     def get_timestamp(self):
         return self.timestamp
@@ -55,11 +58,8 @@ class State2:
     def get_symbology(self):
         return self.symbology
 
-    def get_latitude(self):
-        return self.latitude
-
-    def get_longitude(self):
-        return self.longitude
+    def get_location(self):
+        return (self.latitude, self.longitude)
 
     def get_heading(self):
         return self.heading
diff --git a/pepys_import/core/store/data_store.py b/pepys_import/core/store/data_store.py
index 30c84a50..c07cd8a9 100644
--- a/pepys_import/core/store/data_store.py
+++ b/pepys_import/core/store/data_store.py
@@ -14,6 +14,7 @@ from pepys_import.utils.data_store_utils import import_from_csv
 from .db_base import base_postgres, base_sqlite
 from .db_status import TableTypes
 from pepys_import.core.formats import unit_registry
+from pepys_import.core.formats.state2 import State2
 
 from pepys_import import __version__
 from pepys_import.utils.branding_util import (
@@ -441,8 +442,42 @@ class DataStore:
         # should return DB type or something else decoupled from DB?
         return sensor_obj
 
+    def add_state_to_states(self, state: State2, data_file, sensor):
+        # No cache for entries, just add new one when called
+
+        # don't know privacy, use resolver to query for data
+        privacy = self.missing_data_resolver.resolve_privacy(
+            self,
+            self.db_classes.State.table_type_id,
+            self.db_classes.State.__tablename__,
+        )
+
+        # privacy should contain (table_type, privacy_name)
+        # enough info to proceed and create entry
+        _, privacy = privacy
+        entry_id = self.add_to_entries(
+            self.db_classes.State.table_type_id, self.db_classes.State.__tablename__
+        )
+
+        state_obj = self.db_classes.State(
+            state_id=entry_id,
+            time=state.get_timestamp(),
+            sensor_id=sensor.sensor_id,
+            location=str(state.get_location()),
+            heading=state.get_heading().to(unit_registry.radians).magnitude,
+            speed=state.get_speed()
+            .to(unit_registry.meter / unit_registry.second)
+            .magnitude,
+            datafile_id=data_file.datafile_id,
+            privacy_id=privacy.privacy_id,
+        )
+        self.session.add(state_obj)
+        self.session.flush()
+
+        return state_obj
+
     def add_to_states_from_rep(
-        self, timestamp, datafile, sensor, lat, long, heading, speed
+        self, timestamp, datafile, sensor, location, heading, speed
     ):
         # No cache for entries, just add new one when called
 
@@ -470,7 +505,7 @@ class DataStore:
             state_id=entry_id,
             time=timestamp,
             sensor_id=sensor.sensor_id,
-            location="(" + str(long.degrees) + "," + str(lat.degrees) + ")",
+            location=str(location),
             heading=heading_rads,
             speed=speed_m_sec,
             datafile_id=datafile.datafile_id,
diff --git a/pepys_import/file/file_processor.py b/pepys_import/file/file_processor.py
index 97b2aacf..c4664b65 100644
--- a/pepys_import/file/file_processor.py
+++ b/pepys_import/file/file_processor.py
@@ -11,7 +11,9 @@ class FileProcessor:
         else:
             self.filename = filename
 
-    def process(self, folder: str, data_store: DataStore, descend_tree: bool = True):
+    def process(
+        self, folder: str, data_store: DataStore = None, descend_tree: bool = True
+    ):
         """Process this folder of data
         
         :param folder: Folder path
@@ -129,7 +131,7 @@ class FileProcessor:
 
             # ok, let these parsers handle the file
 
-            with data_store.session_scope() as session:
+            with data_store.session_scope():
                 data_file = data_store.add_to_datafile_from_rep(
                     filename, file_extension
                 )
diff --git a/pepys_import/file/nmea_parser.py b/pepys_import/file/nmea_parser.py
index 1c85ca13..e6ec7e29 100644
--- a/pepys_import/file/nmea_parser.py
+++ b/pepys_import/file/nmea_parser.py
@@ -1,4 +1,7 @@
 from .core_parser import CoreParser
+from pepys_import.core.formats import unit_registry, quantity
+from pepys_import.core.formats.state2 import State2
+from datetime import datetime
 
 
 class NMEAParser(CoreParser):
@@ -17,6 +20,111 @@ class NMEAParser(CoreParser):
     def can_process_file(self, file_contents):
         return True
 
-    def process(self, data_store, path, file_contents, datafile):
+    def process(self, data_store, path, file_contents, data_file_id):
         print("NMEA parser working on " + path)
-        pass
+
+        line_num = 0
+        lat_tok = None
+        lat_hem_tok = None
+        long_tok = None
+        long_hem_tok = None
+        date_tok = None
+        time_tok = None
+        hdg_tok = None
+        spd_tok = None
+
+        ctr = 0
+
+        for line in file_contents:
+
+            ctr += 1
+
+            if ctr > 5000:
+                break
+
+            tokens = line.split(",")
+
+            line_num += 1
+
+            if len(tokens) > 0:
+
+                msg_type = tokens[1]
+                if msg_type == "DZA":
+                    date_tok = tokens[2]
+                    time_tok = tokens[3]
+                elif msg_type == "VEL":
+                    spd_tok = tokens[6]
+                elif msg_type == "HDG":
+                    hdg_tok = tokens[2]
+                elif msg_type == "POS":
+                    lat_tok = tokens[3]
+                    lat_hem_tok = tokens[4]
+                    long_tok = tokens[5]
+                    long_hem_tok = tokens[6]
+
+                # do we have all we need?
+                if date_tok and spd_tok and hdg_tok and lat_tok:
+
+                    # and finally store it
+                    with data_store.session_scope():
+                        datafile = data_store.search_datafile_by_id(data_file_id)
+                        platform = data_store.add_to_platforms_from_rep(
+                            "Toure", "Ferry", "FR", "Public"
+                        )
+                        sensor = data_store.add_to_sensors_from_rep(
+                            platform.name + "_GPS", platform
+                        )
+
+                        date_time = self.parse_timestamp(date_tok, time_tok)
+                        state = State2(date_time, sensor)
+
+                        loc = self.parse_location(
+                            lat_tok, lat_hem_tok, long_tok, long_hem_tok
+                        )
+
+                        state.set_location_obj(loc)
+                        state.set_speed(
+                            float(spd_tok) * unit_registry.metre / unit_registry.second
+                        )
+                        state.set_heading(float(hdg_tok) * unit_registry.degree)
+
+                        data_store.add_state_to_states(
+                            state, datafile, sensor,
+                        )
+
+                        date_tok = None
+                        spd_tok = None
+                        hdg_tok = None
+                        lat_tok = None
+
+    def parse_timestamp(self, date, time):
+        if len(date) == 6:
+            formatStr = "%y%m%d"
+        else:
+            formatStr = "%Y%m%d"
+
+        if len(time) == 6:
+            formatStr += "%H%M%S"
+        else:
+            formatStr += "%H%M%S.%f"
+
+        return datetime.strptime(date + time, formatStr)
+
+    def parse_location(self, lat, lat_hem, lon, long_hem):
+        latDegs = float(lat[0:2])
+        latMins = float(lat[2:4])
+        latSecs = float(lat[4:])
+        latDegs = latDegs + latMins / 60 + latSecs / 60 / 60
+
+        lonDegs = float(lon[0:3])
+        lonMins = float(lon[3:5])
+        lonSecs = float(lon[5:])
+        lonDegs = lonDegs + lonMins / 60 + lonSecs / 60 / 60
+
+        if lat_hem == "S":
+            latDegs = -1 * latDegs
+
+        if lat_hem == "W":
+            lonDegs = -1 * lonDegs
+
+        return (latDegs, lonDegs)
diff --git a/pepys_import/file/replay_parser.py b/pepys_import/file/replay_parser.py
index c032a55b..73efa478 100644
--- a/pepys_import/file/replay_parser.py
+++ b/pepys_import/file/replay_parser.py
@@ -55,7 +55,6 @@ class ReplayParser(CoreParser):
                 heading_token = tokens[12]
                 speed_token = tokens[13]
                 depth_token = tokens[14]
-                text_label_token = ""
 
                 if len(tokens) >= 16:
                     # TODO: join back into single string, or extract full substring
@@ -104,22 +103,16 @@ class ReplayParser(CoreParser):
 
                 new_state.symbology = symbology_token
 
-                new_state.latitude = Location(
+                new_state.latitude = self.degrees_for(
                     lat_degrees_token, lat_mins_token, lat_secs_token, lat_hemi_token
                 )
-                if not new_state.latitude.parse():
-                    print("Line {}. Error in latitude parsing".format(line_num))
-                    return False
 
-                new_state.longitude = Location(
+                new_state.longitude = self.degrees_for(
                     long_degrees_token,
                     long_mins_token,
                     long_secs_token,
                     long_hemi_token,
                 )
-                if not new_state.latitude.parse():
-                    print("Line {}. Error in longitude parsing".format(line_num))
-                    return False
 
                 try:
                     valid_heading = float(heading_token)
@@ -168,7 +161,7 @@ class ReplayParser(CoreParser):
                     return False
 
                 # and finally store it
-                with data_store.session_scope() as session:
+                with data_store.session_scope():
                     datafile = data_store.search_datafile_by_id(data_file_id)
                     platform = data_store.add_to_platforms_from_rep(
                         new_state.get_platform(), "Fisher", "UK", "Public"
@@ -176,16 +169,17 @@ class ReplayParser(CoreParser):
                     sensor = data_store.add_to_sensors_from_rep(
                         platform.name + "_GPS", platform
                     )
-                    data_store.add_to_states_from_rep(
-                        new_state.get_timestamp(),
-                        datafile,
-                        sensor,
-                        new_state.get_latitude(),
-                        new_state.get_longitude(),
-                        new_state.get_heading(),
-                        new_state.get_speed(),
+                    data_store.add_state_to_states(
+                        new_state, datafile, sensor,
                     )
 
+    def degrees_for(self, degs, mins, secs, hemi: str):
+        if hemi.upper() == "S" or hemi.upper() == "W":
+            factor = -1
+        else:
+            factor = 1
+        return factor * (float(degs) + float(mins) / 60 + float(secs) / 60 / 60)
+
     def parse_timestamp(self, date, time):
         if len(date) == 6:
             formatStr = "%y%m%d"


Test patch:
diff --git a/tests/test_replay_load.py b/tests/test_replay_load.py
index dac02225..81ba174f 100644
--- a/tests/test_replay_load.py
+++ b/tests/test_replay_load.py
@@ -43,8 +43,7 @@ class TestLoadReplay(TestCase):
                     repLine.get_timestamp(),
                     datafile,
                     sensor,
-                    repLine.get_latitude(),
-                    repLine.get_longitude(),
+                    (repLine.get_latitude(), repLine.get_longitude()),
                     repLine.get_heading(),
                     repLine.get_speed(),
                 )