"""
Code originally developed for pyEcholab
(https://github.com/CI-CMG/pyEcholab)
by Rick Towler <rick.towler@noaa.gov> at NOAA AFSC.

The code has been modified to handle split-beam data and
channel-transducer structure from different EK80 setups.
"""

import logging
import re
import struct
import sys
import xml.etree.ElementTree as ET
from collections import Counter

import numpy as np

from .ek_date_conversion import nt_to_unix

TCVR_CH_NUM_MATCHER = re.compile(r"\d{6}-\w{1,2}|\w{12}-\w{1,2}")

__all__ = [
    "SimradNMEAParser",
    "SimradDepthParser",
    "SimradBottomParser",
    "SimradAnnotationParser",
    "SimradConfigParser",
    "SimradRawParser",
]

log = logging.getLogger(__name__)

class _SimradDatagramParser(object):
    """"""

    def __init__(self, header_type, header_formats):
        self._id = header_type
        self._headers = header_formats
        self._versions = list(header_formats.keys())

    def header_fmt(self, version=0):
        return "=" + "".join([x[1] for x in self._headers[version]])

    def header_size(self, version=0):
        return struct.calcsize(self.header_fmt(version))

    def header_fields(self, version=0):
        return [x[0] for x in self._headers[version]]

    def header(self, version=0):
        return self._headers[version][:]

    def validate_data_header(self, data):

        if isinstance(data, dict):
            type_ = data["type"][:3]
            version = int(data["type"][3])

        elif isinstance(data, str):
            type_ = data[:3]
            version = int(data[3])

        else:
            raise TypeError("Expected a dict or str")

        if type_ != self._id:
            raise ValueError("Expected data of type %s, not %s" % (self._id, type_))

        if version not in self._versions:
            raise ValueError(
                "No parser available for type %s version %d" % (self._id, version)
            )

        return type_, version

    def from_string(self, raw_string, bytes_read):

        header = raw_string[:4]
        if sys.version_info.major > 2:
            header = header.decode()
        id_, version = self.validate_data_header(header)
        return self._unpack_contents(raw_string, bytes_read, version=version)

    def to_string(self, data={}):

        id_, version = self.validate_data_header(data)
        datagram_content_str = self._pack_contents(data, version=version)
        return self.finalize_datagram(datagram_content_str)

    def _unpack_contents(self, raw_string="", version=0):
        raise NotImplementedError

    def _pack_contents(self, data={}, version=0):
        raise NotImplementedError

    @classmethod
    def finalize_datagram(cls, datagram_content_str):
        datagram_size = len(datagram_content_str)
        final_fmt = "=l%dsl" % (datagram_size)
        return struct.pack(
            final_fmt, datagram_size, datagram_content_str, datagram_size
        )

class SimradDepthParser(_SimradDatagramParser):
    """
    ER60 Depth Detection datagram (from .bot files) contain the following keys:

        type:         string == 'DEP0'
        low_date:     long uint representing LSBytes of 64bit NT date
        high_date:    long uint representing MSBytes of 64bit NT date
        timestamp:    datetime.datetime object of NT date, assumed to be UTC
        transceiver_count:  [long uint] with number of tranceivers

        depth:        [float], one value for each active channel
        reflectivity: [float], one value for each active channel
        unused:       [float], unused value for each active channel

    The following methods are defined:

        from_string(str):    parse a raw ER60 Depth datagram
                             (with leading/trailing datagram size stripped)

        to_string():         Returns the datagram as a raw string
                             (including leading/trailing size fields)
                             ready for writing to disk

    """

    def __init__(self):
        headers = {
            0: [
                ("type", "4s"),
                ("low_date", "L"),
                ("high_date", "L"),
                ("transceiver_count", "L"),
            ]
        }
        _SimradDatagramParser.__init__(self, "DEP", headers)

    def _unpack_contents(self, raw_string, bytes_read, version):
        """"""

        header_values = struct.unpack(
            self.header_fmt(version), raw_string[: self.header_size(version)]
        )
        data = {}

        for indx, field in enumerate(self.header_fields(version)):
            data[field] = header_values[indx]
            if isinstance(data[field], bytes):
                data[field] = data[field].decode()

        data["timestamp"] = nt_to_unix((data["low_date"], data["high_date"]))
        data["bytes_read"] = bytes_read

        if version == 0:
            data_fmt = "=3f"
            data_size = struct.calcsize(data_fmt)

            data["depth"] = np.zeros((data["transceiver_count"],))
            data["reflectivity"] = np.zeros((data["transceiver_count"],))
            data["unused"] = np.zeros((data["transceiver_count"],))

            buf_indx = self.header_size(version)
            for indx in range(data["transceiver_count"]):
                d, r, u = struct.unpack(
                    data_fmt, raw_string[buf_indx : buf_indx + data_size]  # noqa
                )
                data["depth"][indx] = d
                data["reflectivity"][indx] = r
                data["unused"][indx] = u

                buf_indx += data_size

        return data

    def _pack_contents(self, data, version):

        datagram_fmt = self.header_fmt(version)
        datagram_contents = []

        if version == 0:

            lengths = [
                len(data["depth"]),
                len(data["reflectivity"]),
                len(data["unused"]),
                data["transceiver_count"],
            ]

            if len(set(lengths)) != 1:
                min_indx = min(lengths)
                log.warning(
                    "Data lengths mismatched:  d:%d, r:%d, u:%d, t:%d", *lengths
                )
                log.warning("  Using minimum value:  %d", min_indx)
                data["transceiver_count"] = min_indx

            else:
                min_indx = data["transceiver_count"]

            for field in self.header_fields(version):
                datagram_contents.append(data[field])

            datagram_fmt += "%df" % (3 * data["transceiver_count"])

            for indx in range(data["transceiver_count"]):
                datagram_contents.extend(
                    [
                        data["depth"][indx],
                        data["reflectivity"][indx],
                        data["unused"][indx],
                    ]
                )

        return struct.pack(datagram_fmt, *datagram_contents)

class SimradBottomParser(_SimradDatagramParser):
    """
    Bottom Detection datagram contains the following keys:

        type:         string == 'BOT0'
        low_date:     long uint representing LSBytes of 64bit NT date
        high_date:    long uint representing MSBytes of 64bit NT date
        datetime:     datetime.datetime object of NT date converted to UTC
        transceiver_count:  long uint with number of tranceivers
        depth:        [float], one value for each active channel

    The following methods are defined:

        from_string(str):    parse a raw ER60 Bottom datagram
                            (with leading/trailing datagram size stripped)

        to_string():         Returns the datagram as a raw string
                             (including leading/trailing size fields)
                             ready for writing to disk
    """

    def __init__(self):
        headers = {
            0: [
                ("type", "4s"),
                ("low_date", "L"),
                ("high_date", "L"),
                ("transceiver_count", "L"),
            ]
        }
        _SimradDatagramParser.__init__(self, "BOT", headers)

    def _unpack_contents(self, raw_string, bytes_read, version):
        """"""

        header_values = struct.unpack(
            self.header_fmt(version), raw_string[: self.header_size(version)]
        )
        data = {}

        for indx, field in enumerate(self.header_fields(version)):
            data[field] = header_values[indx]
            if isinstance(data[field], bytes):
                data[field] = data[field].decode()

        data["timestamp"] = nt_to_unix((data["low_date"], data["high_date"]))
        data["bytes_read"] = bytes_read

        if version == 0:
            depth_fmt = "=%dd" % (data["transceiver_count"],)
            depth_size = struct.calcsize(depth_fmt)
            buf_indx = self.header_size(version)
            data["depth"] = np.fromiter(
                struct.unpack(
                    depth_fmt, raw_string[buf_indx : buf_indx + depth_size]
                ),  # noqa
                "float",
            )

        return data

    def _pack_contents(self, data, version):

        datagram_fmt = self.header_fmt(version)
        datagram_contents = []

        if version == 0:

            if len(data["depth"]) != data["transceiver_count"]:
                log.warning(
                    "# of depth values %d does not match transceiver count %d",
                    len(data["depth"]),
                    data["transceiver_count"],
                )

                data["transceiver_count"] = len(data["depth"])

            for field in self.header_fields(version):
                datagram_contents.append(data[field])

            datagram_fmt += "%dd" % (data["transceiver_count"])
            datagram_contents.extend(data["depth"])

        return struct.pack(datagram_fmt, *datagram_contents)

class SimradAnnotationParser(_SimradDatagramParser):
    """
    ER60 Annotation datagram contains the following keys:

        type:         string == 'TAG0'
        low_date:     long uint representing LSBytes of 64bit NT date
        high_date:    long uint representing MSBytes of 64bit NT date
        timestamp:     datetime.datetime object of NT date, assumed to be UTC

        text:         Annotation

    The following methods are defined:

        from_string(str):    parse a raw ER60 Annotation datagram
                            (with leading/trailing datagram size stripped)

        to_string():         Returns the datagram as a raw string
                             (including leading/trailing size fields)
                             ready for writing to disk
    """

    def __init__(self):
        headers = {0: [("type", "4s"), ("low_date", "L"), ("high_date", "L")]}

        _SimradDatagramParser.__init__(self, "TAG", headers)

    def _unpack_contents(self, raw_string, bytes_read, version):
        """"""

        header_values = struct.unpack(
            self.header_fmt(version), raw_string[: self.header_size(version)]
        )
        data = {}

        for indx, field in enumerate(self.header_fields(version)):
            data[field] = header_values[indx]
            if isinstance(data[field], bytes):
                data[field] = data[field].decode()

        data["timestamp"] = nt_to_unix((data["low_date"], data["high_date"]))
        data["bytes_read"] = bytes_read

        #        if version == 0:
        #            data['text'] = raw_string[self.header_size(version):].strip('\x00')
        #            if isinstance(data['text'], bytes):
        #                data['text'] = data['text'].decode()

        if version == 0:
            if sys.version_info.major > 2:
                data["text"] = str(
                    raw_string[self.header_size(version) :].strip(b"\x00"),
                    "ascii",
                    errors="replace",
                )
            else:
                data["text"] = unicode(  # noqa
                    raw_string[self.header_size(version) :].strip("\x00"),
                    "ascii",
                    errors="replace",
                )

        return data

    def _pack_contents(self, data, version):

        datagram_fmt = self.header_fmt(version)
        datagram_contents = []

        if version == 0:

            for field in self.header_fields(version):
                datagram_contents.append(data[field])

            if data["text"][-1] != "\x00":
                tmp_string = data["text"] + "\x00"
            else:
                tmp_string = data["text"]

            # Pad with more nulls to 4-byte word boundry if necessary
            if len(tmp_string) % 4:
                tmp_string += "\x00" * (4 - (len(tmp_string) % 4))

            datagram_fmt += "%ds" % (len(tmp_string))
            datagram_contents.append(tmp_string)

        return struct.pack(datagram_fmt, *datagram_contents)

class SimradNMEAParser(_SimradDatagramParser):
    """
    ER60 NMEA datagram contains the following keys:

        type:         string == 'NME0'
        low_date:     long uint representing LSBytes of 64bit NT date
        high_date:    long uint representing MSBytes of 64bit NT date
        timestamp:     datetime.datetime object of NT date, assumed to be UTC

        nmea_string:  full (original) NMEA string

    The following methods are defined:

        from_string(str):    parse a raw ER60 NMEA datagram
                            (with leading/trailing datagram size stripped)

        to_string():         Returns the datagram as a raw string
                             (including leading/trailing size fields)
                             ready for writing to disk
    """

    nmea_head_re = re.compile(r"\$[A-Za-z]{5},")  # noqa

    def __init__(self):
        headers = {
            0: [("type", "4s"), ("low_date", "L"), ("high_date", "L")],
            1: [("type", "4s"), ("low_date", "L"), ("high_date", "L"), ("port", "32s")],
        }

        _SimradDatagramParser.__init__(self, "NME", headers)

    def _unpack_contents(self, raw_string, bytes_read, version):
        """
        Parses the NMEA string provided in raw_string

        :param raw_string:  Raw NMEA strin (i.e. '$GPZDA,160012.71,11,03,2004,-1,00*7D')
        :type raw_string: str

        :returns: None
        """

        header_values = struct.unpack(
            self.header_fmt(version), raw_string[: self.header_size(version)]
        )
        data = {}

        for indx, field in enumerate(self.header_fields(version)):
            data[field] = header_values[indx]
            if isinstance(data[field], bytes):
                data[field] = data[field].decode()

        data["timestamp"] = nt_to_unix((data["low_date"], data["high_date"]))
        data["bytes_read"] = bytes_read

        # Remove trailing \x00 from the PORT field for NME1, rest of the datagram identical to NME0
        if version == 1:
            data["port"] = data["port"].strip("\x00")

        if version == 0 or version == 1:
            if sys.version_info.major > 2:
                data["nmea_string"] = str(
                    raw_string[self.header_size(version) :].strip(b"\x00"),
                    "ascii",
                    errors="replace",
                )
            else:
                data["nmea_string"] = unicode(  # noqa
                    raw_string[self.header_size(version) :].strip("\x00"),
                    "ascii",
                    errors="replace",
                )

            if self.nmea_head_re.match(data["nmea_string"][:7]) is not None:
                data["nmea_talker"] = data["nmea_string"][1:3]
                data["nmea_type"] = data["nmea_string"][3:6]
            else:
                data["nmea_talker"] = ""
                data["nmea_type"] = "UNKNOWN"

        return data

    def _pack_contents(self, data, version):

        datagram_fmt = self.header_fmt(version)
        datagram_contents = []

        if version == 0:

            for field in self.header_fields(version):
                datagram_contents.append(data[field])

            if data["nmea_string"][-1] != "\x00":
                tmp_string = data["nmea_string"] + "\x00"
            else:
                tmp_string = data["nmea_string"]

            # Pad with more nulls to 4-byte word boundry if necessary
            if len(tmp_string) % 4:
                tmp_string += "\x00" * (4 - (len(tmp_string) % 4))

            datagram_fmt += "%ds" % (len(tmp_string))

            # Convert to python string if needed
            if isinstance(tmp_string, str):
                tmp_string = tmp_string.encode("ascii", errors="replace")

            datagram_contents.append(tmp_string)

        return struct.pack(datagram_fmt, *datagram_contents)

class SimradMRUParser(_SimradDatagramParser):
    """
    EK80 MRU datagram contains the following keys:

        type:         string == 'MRU0'
        low_date:     long uint representing LSBytes of 64bit NT date
        high_date:    long uint representing MSBytes of 64bit NT date
        timestamp:    datetime.datetime object of NT date, assumed to be UTC
        heave:        float
        roll :        float
        pitch:        float
        heading:      float

    The following methods are defined:

        from_string(str):    parse a raw ER60 NMEA datagram
                            (with leading/trailing datagram size stripped)

        to_string():         Returns the datagram as a raw string
                             (including leading/trailing size fields)
                             ready for writing to disk
    """

    def __init__(self):
        headers = {
            0: [
                ("type", "4s"),
                ("low_date", "L"),
                ("high_date", "L"),
                ("heave", "f"),
                ("roll", "f"),
                ("pitch", "f"),
                ("heading", "f"),
            ]
        }

        _SimradDatagramParser.__init__(self, "MRU", headers)

    def _unpack_contents(self, raw_string, bytes_read, version):
        """
        Unpacks the data in raw_string into dictionary containing MRU data

        :param raw_string:
        :type raw_string: str

        :returns: None
        """

        header_values = struct.unpack(
            self.header_fmt(version), raw_string[: self.header_size(version)]
        )
        data = {}

        for indx, field in enumerate(self.header_fields(version)):
            data[field] = header_values[indx]
            if isinstance(data[field], bytes):
                data[field] = data[field].decode()

        data["timestamp"] = nt_to_unix((data["low_date"], data["high_date"]))
        data["bytes_read"] = bytes_read

        return data

    def _pack_contents(self, data, version):

        datagram_fmt = self.header_fmt(version)
        datagram_contents = []

        if version == 0:

            for field in self.header_fields(version):
                datagram_contents.append(data[field])

            if data["nmea_string"][-1] != "\x00":
                tmp_string = data["nmea_string"] + "\x00"
            else:
                tmp_string = data["nmea_string"]

            # Pad with more nulls to 4-byte word boundry if necessary
            if len(tmp_string) % 4:
                tmp_string += "\x00" * (4 - (len(tmp_string) % 4))

            datagram_fmt += "%ds" % (len(tmp_string))

            # Convert to python string if needed
            if isinstance(tmp_string, str):
                tmp_string = tmp_string.encode("ascii", errors="replace")

            datagram_contents.append(tmp_string)

        return struct.pack(datagram_fmt, *datagram_contents)

class SimradXMLParser(_SimradDatagramParser):
    """
    EK80 XML datagram contains the following keys:

        type:         string == 'XML0'
        low_date:     long uint representing LSBytes of 64bit NT date
        high_date:    long uint representing MSBytes of 64bit NT date
        timestamp:    datetime.datetime object of NT date, assumed to be UTC
        subtype:      string representing Simrad XML datagram type:
                      configuration, environment, or parameter

        [subtype]:    dict containing the data specific to the XML subtype.

    The following methods are defined:

        from_string(str):    parse a raw EK80 XML datagram
                            (with leading/trailing datagram size stripped)

        to_string():         Returns the datagram as a raw string
                             (including leading/trailing size fields)
                             ready for writing to disk
    """

    #  define the XML parsing options - here we define dictionaries for various xml datagram
    #  types. When parsing that xml datagram, these dictionaries are used to inform the parser about
    #  type conversion, name wrangling, and delimiter. If a field is missing, the parser
    #  assumes no conversion: type will be string, default mangling, and that there is only 1
    #  element.
    #
    #  the dicts are in the form:
    #       'XMLParamName':[converted type,'fieldname', 'parse char']
    #
    #  For example: 'PulseDurationFM':[float,'pulse_duration_fm',';']
    #
    #  will result in a return dictionary field named 'pulse_duration_fm' that contains a list
    #  of float values parsed from a string that uses ';' to separate values. Empty strings
    #  for fieldname and/or parse char result in the default action for those parsing steps.

    channel_parsing_options = {
        "MaxTxPowerTransceiver": [int, "", ""],
        "PulseDuration": [float, "", ";"],
        "PulseDurationFM": [float, "pulse_duration_fm", ";"],
        "SampleInterval": [float, "", ";"],
        "ChannelID": [str, "channel_id", ""],
        "HWChannelConfiguration": [str, "hw_channel_configuration", ""],
    }

    transceiver_parsing_options = {
        "TransceiverNumber": [int, "", ""],
        "Version": [str, "transceiver_version", ""],
        "IPAddress": [str, "ip_address", ""],
        "Impedance": [int, "", ""],
    }

    transducer_parsing_options = {
        "SerialNumber": [str, "transducer_serial_number", ""],
        "Frequency": [float, "transducer_frequency", ""],
        "FrequencyMinimum": [float, "transducer_frequency_minimum", ""],
        "FrequencyMaximum": [float, "transducer_frequency_maximum", ""],
        "BeamType": [int, "transducer_beam_type", ""],
        "Gain": [float, "", ";"],
        "SaCorrection": [float, "", ";"],
        "MaxTxPowerTransducer": [float, "", ""],
        "EquivalentBeamAngle": [float, "", ""],
        "BeamWidthAlongship": [float, "", ""],
        "BeamWidthAthwartship": [float, "", ""],
        "AngleSensitivityAlongship": [float, "", ""],
        "AngleSensitivityAthwartship": [float, "", ""],
        "AngleOffsetAlongship": [float, "", ""],
        "AngleOffsetAthwartship": [float, "", ""],
        "DirectivityDropAt2XBeamWidth": [
            float,
            "directivity_drop_at_2x_beam_width",
            "",
        ],
        "TransducerOffsetX": [float, "", ""],
        "TransducerOffsetY": [float, "", ""],
        "TransducerOffsetZ": [float, "", ""],
        "TransducerAlphaX": [float, "", ""],
        "TransducerAlphaY": [float, "", ""],
        "TransducerAlphaZ": [float, "", ""],
    }

    header_parsing_options = {"Version": [str, "application_version", ""]}

    envxdcr_parsing_options = {"SoundSpeed": [float, "transducer_sound_speed", ""]}

    environment_parsing_options = {
        "Depth": [float, "", ""],
        "Acidity": [float, "", ""],
        "Salinity": [float, "", ""],
        "SoundSpeed": [float, "", ""],
        "Temperature": [float, "", ""],
        "Latitude": [float, "", ""],
        "SoundVelocityProfile": [float, "", ";"],
        "DropKeelOffset": [float, "", ""],
        "DropKeelOffsetIsManual": [int, "", ""],
        "WaterLevelDraft": [float, "", ""],
        "WaterLevelDraftIsManual": [int, "", ""],
    }

    parameter_parsing_options = {
        "ChannelID": [str, "channel_id", ""],
        "ChannelMode": [int, "", ""],
        "PulseForm": [int, "", ""],
        "Frequency": [float, "", ""],
        "PulseDuration": [float, "", ""],
        "SampleInterval": [float, "", ""],
        "TransmitPower": [float, "", ""],
        "Slope": [float, "", ""],
    }

    def __init__(self):
        headers = {0: [("type", "4s"), ("low_date", "L"), ("high_date", "L")]}
        _SimradDatagramParser.__init__(self, "XML", headers)

    def _unpack_contents(self, raw_string, bytes_read, version):
        """
        Parses the NMEA string provided in raw_string

        :param raw_string:  Raw NMEA strin (i.e. '$GPZDA,160012.71,11,03,2004,-1,00*7D')
        :type raw_string: str

        :returns: None
        """

        def from_CamelCase(xml_param):
            """
            convert name from CamelCase to fit with existing naming convention by
            inserting an underscore before each capital and then lowering the caps
            e.g. CamelCase becomes camel_case.
            """
            idx = list(reversed([i for i, c in enumerate(xml_param) if c.isupper()]))
            param_len = len(xml_param)
            for i in idx:
                #  check if we should insert an underscore
                if i > 0 and i < param_len:
                    xml_param = xml_param[:i] + "_" + xml_param[i:]
            xml_param = xml_param.lower()

            return xml_param

        def dict_to_dict(xml_dict, data_dict, parse_opts):
            """
            dict_to_dict appends the ETree xml value dicts to a provided dictionary
            and along the way converts the key name to conform to the project's
            naming convention and optionally parses and or converts values as
            specified in the parse_opts dictionary.
            """

            for k in xml_dict:
                #  check if we're parsing this key/value
                if k in parse_opts:
                    #  try to parse the string
                    if parse_opts[k][2]:
                        try:
                            data = xml_dict[k].split(parse_opts[k][2])
                        except:
                            #  bad or empty parse chararacter(s) provided
                            data = xml_dict[k]
                    else:
                        #  no parse char provided - nothing to parse
                        data = xml_dict[k]

                    #  try to convert to specified type
                    if isinstance(data, list):
                        for i in range(len(data)):
                            try:
                                data[i] = parse_opts[k][0](data[i])
                            except:
                                pass
                    else:
                        data = parse_opts[k][0](data)

                    #  and add the value to the provided dict
                    if parse_opts[k][1]:
                        #  add using the specified key name
                        data_dict[parse_opts[k][1]] = data
                    else:
                        #  add using the default key name wrangling
                        data_dict[from_CamelCase(k)] = data
                else:
                    #  nothing to do with the value string
                    data = xml_dict[k]

                    #  add the parameter to the provided dictionary
                    data_dict[from_CamelCase(k)] = data

        header_values = struct.unpack(
            self.header_fmt(version), raw_string[: self.header_size(version)]
        )
        data = {}

        for indx, field in enumerate(self.header_fields(version)):
            data[field] = header_values[indx]
            if isinstance(data[field], bytes):
                data[field] = data[field].decode()

        data["timestamp"] = nt_to_unix((data["low_date"], data["high_date"]))
        data["bytes_read"] = bytes_read

        if version == 0:
            if sys.version_info.major > 2:
                xml_string = str(
                    raw_string[self.header_size(version) :].strip(b"\x00"),
                    "ascii",
                    errors="replace",
                )
            else:
                xml_string = unicode(  # noqa
                    raw_string[self.header_size(version) :].strip("\x00"),
                    "ascii",
                    errors="replace",
                )

            #  get the ElementTree element
            root = ET.fromstring(xml_string)

            #  get the XML message type
            data["subtype"] = root.tag.lower()

            #  create the dictionary that contains the message data
            data[data["subtype"]] = {}

            #  parse it
            if data["subtype"] == "configuration":

                #  parse the Transceiver section
                for tcvr in root.iter("Transceiver"):
                    #  parse the Transceiver section
                    tcvr_xml = tcvr.attrib

                    #  parse the Channel section -- this works with multiple channels
                    #  under 1 transceiver
                    for tcvr_ch in tcvr.iter("Channel"):
                        tcvr_ch_xml = tcvr_ch.attrib
                        channel_id = tcvr_ch_xml["ChannelID"]

                        #  create the configuration dict for this channel
                        data["configuration"][channel_id] = {}

                        #  add the transceiver data to the config dict (this is
                        #  replicated for all channels)
                        dict_to_dict(
                            tcvr_xml,
                            data["configuration"][channel_id],
                            self.transceiver_parsing_options,
                        )

                        #  add the general channel data to the config dict
                        dict_to_dict(
                            tcvr_ch_xml,
                            data["configuration"][channel_id],
                            self.channel_parsing_options,
                        )

                        #  check if there are >1 transducer under a single transceiver channel
                        if len(list(tcvr_ch)) > 1:
                            ValueError(
                                "Found >1 transducer under a single transceiver channel!"
                            )
                        else:  # should only have 1 transducer
                            tcvr_ch_xducer = tcvr_ch.find(
                                "Transducer"
                            )  # get Element of this xducer
                            f_par = tcvr_ch_xducer.findall("FrequencyPar")
                            # Save calibration parameters
                            if f_par:
                                cal_par = {
                                    "frequency": np.array(
                                        [int(f.attrib["Frequency"]) for f in f_par]
                                    ),
                                    "gain": np.array(
                                        [float(f.attrib["Gain"]) for f in f_par]
                                    ),
                                    "impedance": np.array(
                                        [int(f.attrib["Impedance"]) for f in f_par]
                                    ),
                                    "phase": np.array(
                                        [float(f.attrib["Phase"]) for f in f_par]
                                    ),
                                    "beamwidth_alongship": np.array(
                                        [
                                            float(f.attrib["BeamWidthAlongship"])
                                            for f in f_par
                                        ]
                                    ),
                                    "beamwidth_athwartship": np.array(
                                        [
                                            float(f.attrib["BeamWidthAthwartship"])
                                            for f in f_par
                                        ]
                                    ),
                                    "angle_offset_alongship": np.array(
                                        [
                                            float(f.attrib["AngleOffsetAlongship"])
                                            for f in f_par
                                        ]
                                    ),
                                    "angle_offset_athwartship": np.array(
                                        [
                                            float(f.attrib["AngleOffsetAthwartship"])
                                            for f in f_par
                                        ]
                                    ),
                                }
                                data["configuration"][channel_id][
                                    "calibration"
                                ] = cal_par
                            #  add the transducer data to the config dict
                            dict_to_dict(
                                tcvr_ch_xducer.attrib,
                                data["configuration"][channel_id],
                                self.transducer_parsing_options,
                            )

                        # get unique transceiver channel number stored in channel_id
                        tcvr_ch_num = TCVR_CH_NUM_MATCHER.search(channel_id)[0]

                        # parse the Transducers section from the root
                        # TODO Remove Transducers if doesnt exist
                        xducer = root.find("Transducers")
                        if xducer is not None:
                            # built occurrence lookup table for transducer name
                            xducer_name_list = []
                            for xducer_ch in xducer.iter("Transducer"):
                                xducer_name_list.append(
                                    xducer_ch.attrib["TransducerName"]
                                )

                            # find matching transducer for this channel_id
                            match_found = False
                            for xducer_ch in xducer.iter("Transducer"):
                                if not match_found:
                                    xducer_ch_xml = xducer_ch.attrib
                                    match_name = (
                                        xducer_ch.attrib["TransducerName"]
                                        == tcvr_ch_xducer.attrib["TransducerName"]
                                    )
                                    if xducer_ch.attrib["TransducerSerialNumber"] == "":
                                        match_sn = False
                                    else:
                                        match_sn = (
                                            xducer_ch.attrib["TransducerSerialNumber"]
                                            == tcvr_ch_xducer.attrib["SerialNumber"]
                                        )
                                    match_tcvr = (
                                        tcvr_ch_num
                                        in xducer_ch.attrib["TransducerCustomName"]
                                    )

                                    # if find match add the transducer mounting details
                                    if (
                                        Counter(xducer_name_list)[
                                            xducer_ch.attrib["TransducerName"]
                                        ]
                                        > 1
                                    ):
                                        # if more than one transducer has the same name
                                        # only check sn and transceiver unique number
                                        match_found = match_sn or match_tcvr
                                    else:
                                        match_found = (
                                            match_name or match_sn or match_tcvr
                                        )

                                    # add transducer mounting details
                                    if match_found:
                                        dict_to_dict(
                                            xducer_ch_xml,
                                            data["configuration"][channel_id],
                                            self.transducer_parsing_options,
                                        )

                        #  add the header data to the config dict
                        h = root.find("Header")
                        dict_to_dict(
                            h.attrib,
                            data["configuration"][channel_id],
                            self.header_parsing_options,
                        )

            elif data["subtype"] == "parameter":

                #  parse the parameter XML datagram
                for h in root.iter("Channel"):
                    parm_xml = h.attrib
                    #  add the data to the environment dict
                    dict_to_dict(
                        parm_xml, data["parameter"], self.parameter_parsing_options
                    )

            elif data["subtype"] == "environment":

                #  parse the environment XML datagram
                for h in root.iter("Environment"):
                    env_xml = h.attrib
                    #  add the data to the environment dict
                    dict_to_dict(
                        env_xml, data["environment"], self.environment_parsing_options
                    )

                for h in root.iter("Transducer"):
                    transducer_xml = h.attrib
                    #  add the data to the environment dict
                    dict_to_dict(
                        transducer_xml,
                        data["environment"],
                        self.envxdcr_parsing_options,
                    )

        data["xml"] = xml_string
        return data

    def _pack_contents(self, data, version):
        def to_CamelCase(xml_param):
            """
            convert name from project's convention to CamelCase for converting back to
            XML to in Kongsberg's convention.
            """
            idx = list(reversed([i for i, c in enumerate(xml_param) if c.isupper()]))
            param_len = len(xml_param)
            for i in idx:
                #  check if we should insert an underscore
                if idx > 0 and idx < param_len - 1:
                    xml_param = xml_param[:idx] + "_" + xml_param[idx:]
            xml_param = xml_param.lower()

            return xml_param

        datagram_fmt = self.header_fmt(version)
        datagram_contents = []

        if version == 0:

            for field in self.header_fields(version):
                datagram_contents.append(data[field])

            if data["nmea_string"][-1] != "\x00":
                tmp_string = data["nmea_string"] + "\x00"
            else:
                tmp_string = data["nmea_string"]

            # Pad with more nulls to 4-byte word boundry if necessary
            if len(tmp_string) % 4:
                tmp_string += "\x00" * (4 - (len(tmp_string) % 4))

            datagram_fmt += "%ds" % (len(tmp_string))

            # Convert to python string if needed
            if isinstance(tmp_string, str):
                tmp_string = tmp_string.encode("ascii", errors="replace")

            datagram_contents.append(tmp_string)

        return struct.pack(datagram_fmt, *datagram_contents)

class SimradFILParser(_SimradDatagramParser):
    """
    EK80 FIL datagram contains the following keys:

        type:               string == 'FIL1'
        low_date:           long uint representing LSBytes of 64bit NT date
        high_date:          long uint representing MSBytes of 64bit NT date
        timestamp:          datetime.datetime object of NT date, assumed to be UTC
        stage:              int
        channel_id:         string
        n_coefficients:     int
        decimation_factor:  int
        coefficients:       np.complex64

    The following methods are defined:

        from_string(str):    parse a raw EK80 FIL datagram
                            (with leading/trailing datagram size stripped)

        to_string():         Returns the datagram as a raw string
                            (including leading/trailing size fields)
                             ready for writing to disk
    """

    def __init__(self):
        headers = {
            1: [
                ("type", "4s"),
                ("low_date", "L"),
                ("high_date", "L"),
                ("stage", "h"),
                ("spare", "2s"),
                ("channel_id", "128s"),
                ("n_coefficients", "h"),
                ("decimation_factor", "h"),
            ]
        }

        _SimradDatagramParser.__init__(self, "FIL", headers)

    def _unpack_contents(self, raw_string, bytes_read, version):

        data = {}
        header_values = struct.unpack(
            self.header_fmt(version), raw_string[: self.header_size(version)]
        )

        for indx, field in enumerate(self.header_fields(version)):
            data[field] = header_values[indx]

            #  handle Python 3 strings
            if (sys.version_info.major > 2) and isinstance(data[field], bytes):
                data[field] = data[field].decode("latin_1")

        data["timestamp"] = nt_to_unix((data["low_date"], data["high_date"]))
        data["bytes_read"] = bytes_read

        if version == 1:
            #  clean up the channel ID
            data["channel_id"] = data["channel_id"].strip("\x00")

            #  unpack the coefficients
            indx = self.header_size(version)
            block_size = data["n_coefficients"] * 8
            data["coefficients"] = np.frombuffer(
                raw_string[indx : indx + block_size], dtype="complex64"  # noqa
            )

        return data

    def _pack_contents(self, data, version):

        datagram_fmt = self.header_fmt(version)
        datagram_contents = []

        if version == 0:

            pass

        elif version == 1:
            for field in self.header_fields(version):
                datagram_contents.append(data[field])

            datagram_fmt += "%ds" % (len(data["beam_config"]))
            datagram_contents.append(data["beam_config"])

        return struct.pack(datagram_fmt, *datagram_contents)

class SimradConfigParser(_SimradDatagramParser):
    """
    Simrad Configuration Datagram parser operates on dictionaries with the following keys:

        type:         string == 'CON0'
        low_date:     long uint representing LSBytes of 64bit NT date
        high_date:    long uint representing MSBytes of 64bit NT date
        timestamp:    datetime.datetime object of NT date, assumed to be UTC

        survey_name                     [str]
        transect_name                   [str]
        sounder_name                    [str]
        version                         [str]
        spare0                          [str]
        transceiver_count               [long]
        transceivers                    [list] List of dicts representing Transducer Configs:

        ME70 Data contains the following additional values (data contained w/in first 14
            bytes of the spare0 field)

        multiplexing                    [short]  Always 0
        time_bias                       [long] difference between UTC and local time in min.
        sound_velocity_avg              [float] [m/s]
        sound_velocity_transducer       [float] [m/s]
        beam_config                     [str] Raw XML string containing beam config. info

    Transducer Config Keys (ER60/ES60/ES70 sounders):
        channel_id                      [str]   channel ident string
        beam_type                       [long]  Type of channel (0 = Single, 1 = Split)
        frequency                       [float] channel frequency
        equivalent_beam_angle           [float] dB
        beamwidth_alongship             [float]
        beamwidth_athwartship           [float]
        angle_sensitivity_alongship     [float]
        angle_sensitivity_athwartship   [float]
        angle_offset_alongship          [float]
        angle_offset_athwartship        [float]
        pos_x                           [float]
        pos_y                           [float]
        pos_z                           [float]
        dir_x                           [float]
        dir_y                           [float]
        dir_z                           [float]
        pulse_length_table              [float[5]]
        spare1                          [str]
        gain_table                      [float[5]]
        spare2                          [str]
        sa_correction_table             [float[5]]
        spare3                          [str]
        gpt_software_version            [str]
        spare4                          [str]

    Transducer Config Keys (ME70 sounders):
        channel_id                      [str]   channel ident string
        beam_type                       [long]  Type of channel (0 = Single, 1 = Split)
        reserved1                       [float] channel frequency
        equivalent_beam_angle           [float] dB
        beamwidth_alongship             [float]
        beamwidth_athwartship           [float]
        angle_sensitivity_alongship     [float]
        angle_sensitivity_athwartship   [float]
        angle_offset_alongship          [float]
        angle_offset_athwartship        [float]
        pos_x                           [float]
        pos_y                           [float]
        pos_z                           [float]
        beam_steering_angle_alongship   [float]
        beam_steering_angle_athwartship [float]
        beam_steering_angle_unused      [float]
        pulse_length                    [float]
        reserved2                       [float]
        spare1                          [str]
        gain                            [float]
        reserved3                       [float]
        spare2                          [str]
        sa_correction                   [float]
        reserved4                       [float]
        spare3                          [str]
        gpt_software_version            [str]
        spare4                          [str]

    from_string(str):   parse a raw config datagram
                        (with leading/trailing datagram size stripped)

    to_string(dict):    Returns raw string (including leading/trailing size fields)
                        ready for writing to disk
    """

    COMMON_KEYS = [
        ("channel_id", "128s"),
        ("beam_type", "l"),
        ("frequency", "f"),
        ("gain", "f"),
        ("equivalent_beam_angle", "f"),
        ("beamwidth_alongship", "f"),
        ("beamwidth_athwartship", "f"),
        ("angle_sensitivity_alongship", "f"),
        ("angle_sensitivity_athwartship", "f"),
        ("angle_offset_alongship", "f"),
        ("angle_offset_athwartship", "f"),
        ("pos_x", "f"),
        ("pos_y", "f"),
        ("pos_z", "f"),
        ("dir_x", "f"),
        ("dir_y", "f"),
        ("dir_z", "f"),
        ("pulse_length_table", "5f"),
        ("spare1", "8s"),
        ("gain_table", "5f"),
        ("spare2", "8s"),
        ("sa_correction_table", "5f"),
        ("spare3", "8s"),
        ("gpt_software_version", "16s"),
        ("spare4", "28s"),
    ]

    def __init__(self):
        headers = {
            0: [
                ("type", "4s"),
                ("low_date", "L"),
                ("high_date", "L"),
                ("survey_name", "128s"),
                ("transect_name", "128s"),
                ("sounder_name", "128s"),
                ("version", "30s"),
                ("spare0", "98s"),
                ("transceiver_count", "l"),
            ],
            1: [("type", "4s"), ("low_date", "L"), ("high_date", "L")],
        }

        _SimradDatagramParser.__init__(self, "CON", headers)

        self._transducer_headers = {
            "ER60": self.COMMON_KEYS,
            "ES60": self.COMMON_KEYS,
            "ES70": self.COMMON_KEYS,
            "MBES": [
                ("channel_id", "128s"),
                ("beam_type", "l"),
                ("frequency", "f"),
                ("reserved1", "f"),
                ("equivalent_beam_angle", "f"),
                ("beamwidth_alongship", "f"),
                ("beamwidth_athwartship", "f"),
                ("angle_sensitivity_alongship", "f"),
                ("angle_sensitivity_athwartship", "f"),
                ("angle_offset_alongship", "f"),
                ("angle_offset_athwartship", "f"),
                ("pos_x", "f"),
                ("pos_y", "f"),
                ("pos_z", "f"),
                ("beam_steering_angle_alongship", "f"),
                ("beam_steering_angle_athwartship", "f"),
                ("beam_steering_angle_unused", "f"),
                ("pulse_length", "f"),
                ("reserved2", "f"),
                ("spare1", "20s"),
                ("gain", "f"),
                ("reserved3", "f"),
                ("spare2", "20s"),
                ("sa_correction", "f"),
                ("reserved4", "f"),
                ("spare3", "20s"),
                ("gpt_software_version", "16s"),
                ("spare4", "28s"),
            ],
        }

    def _unpack_contents(self, raw_string, bytes_read, version):

        data = {}
        round6 = lambda x: round(x, ndigits=6)  # noqa
        header_values = struct.unpack(
            self.header_fmt(version), raw_string[: self.header_size(version)]
        )

        for indx, field in enumerate(self.header_fields(version)):
            data[field] = header_values[indx]

            #  handle Python 3 strings
            if (sys.version_info.major > 2) and isinstance(data[field], bytes):
                data[field] = data[field].decode("latin_1")

        data["timestamp"] = nt_to_unix((data["low_date"], data["high_date"]))
        data["bytes_read"] = bytes_read

        if version == 0:

            data["transceivers"] = {}

            for field in ["transect_name", "version", "survey_name", "sounder_name"]:
                data[field] = data[field].strip("\x00")

            sounder_name = data["sounder_name"]
            if sounder_name == "MBES":
                _me70_extra_values = struct.unpack("=hLff", data["spare0"][:14])
                data["multiplexing"] = _me70_extra_values[0]
                data["time_bias"] = _me70_extra_values[1]
                data["sound_velocity_avg"] = _me70_extra_values[2]
                data["sound_velocity_transducer"] = _me70_extra_values[3]
                data["spare0"] = data["spare0"][:14] + data["spare0"][14:].strip("\x00")

            else:
                data["spare0"] = data["spare0"].strip("\x00")

            buf_indx = self.header_size(version)

            try:
                transducer_header = self._transducer_headers[sounder_name]
                _sounder_name_used = sounder_name
            except KeyError:
                log.warning(
                    "Unknown sounder_name:  %s, (no one of %s)",
                    sounder_name,
                    list(self._transducer_headers.keys()),
                )
                log.warning("Will use ER60 transducer config fields as default")

                transducer_header = self._transducer_headers["ER60"]
                _sounder_name_used = "ER60"

            txcvr_header_fields = [x[0] for x in transducer_header]
            txcvr_header_fmt = "=" + "".join([x[1] for x in transducer_header])
            txcvr_header_size = struct.calcsize(txcvr_header_fmt)

            for txcvr_indx in range(1, data["transceiver_count"] + 1):
                txcvr_header_values_encoded = struct.unpack(
                    txcvr_header_fmt,
                    raw_string[buf_indx : buf_indx + txcvr_header_size],  # noqa
                )
                txcvr_header_values = list(txcvr_header_values_encoded)
                for tx_idx, tx_val in enumerate(txcvr_header_values_encoded):
                    if isinstance(tx_val, bytes):
                        txcvr_header_values[tx_idx] = tx_val.decode("latin_1")

                txcvr = data["transceivers"].setdefault(txcvr_indx, {})

                if _sounder_name_used in ["ER60", "ES60", "ES70"]:
                    for txcvr_field_indx, field in enumerate(txcvr_header_fields[:17]):
                        txcvr[field] = txcvr_header_values[txcvr_field_indx]

                    txcvr["pulse_length_table"] = np.fromiter(
                        list(map(round6, txcvr_header_values[17:22])), "float"
                    )
                    txcvr["spare1"] = txcvr_header_values[22]
                    txcvr["gain_table"] = np.fromiter(
                        list(map(round6, txcvr_header_values[23:28])), "float"
                    )
                    txcvr["spare2"] = txcvr_header_values[28]
                    txcvr["sa_correction_table"] = np.fromiter(
                        list(map(round6, txcvr_header_values[29:34])), "float"
                    )
                    txcvr["spare3"] = txcvr_header_values[34]
                    txcvr["gpt_software_version"] = txcvr_header_values[35]
                    txcvr["spare4"] = txcvr_header_values[36]

                elif _sounder_name_used == "MBES":
                    for txcvr_field_indx, field in enumerate(txcvr_header_fields):
                        txcvr[field] = txcvr_header_values[txcvr_field_indx]

                else:
                    raise RuntimeError(
                        "Unknown _sounder_name_used (Should not happen, this is a bug!)"
                    )

                txcvr["channel_id"] = txcvr["channel_id"].strip("\x00")
                txcvr["spare1"] = txcvr["spare1"].strip("\x00")
                txcvr["spare2"] = txcvr["spare2"].strip("\x00")
                txcvr["spare3"] = txcvr["spare3"].strip("\x00")
                txcvr["spare4"] = txcvr["spare4"].strip("\x00")
                txcvr["gpt_software_version"] = txcvr["gpt_software_version"].strip(
                    "\x00"
                )

                buf_indx += txcvr_header_size

        elif version == 1:
            # CON1 only has a single data field:  beam_config, holding an xml string
            data["beam_config"] = raw_string[self.header_size(version) :].strip("\x00")

        return data

    def _pack_contents(self, data, version):

        datagram_fmt = self.header_fmt(version)
        datagram_contents = []

        if version == 0:

            if data["transceiver_count"] != len(data["transceivers"]):
                log.warning(
                    "Mismatch between 'transceiver_count' and actual # of transceivers"
                )
                data["transceiver_count"] = len(data["transceivers"])

            sounder_name = data["sounder_name"]
            if sounder_name == "MBES":
                _packed_me70_values = struct.pack(
                    "=hLff",
                    data["multiplexing"],
                    data["time_bias"],
                    data["sound_velocity_avg"],
                    data["sound_velocity_transducer"],
                )
                data["spare0"] = _packed_me70_values + data["spare0"][14:]

            for field in self.header_fields(version):
                datagram_contents.append(data[field])

            try:
                transducer_header = self._transducer_headers[sounder_name]
                _sounder_name_used = sounder_name
            except KeyError:
                log.warning(
                    "Unknown sounder_name:  %s, (no one of %s)",
                    sounder_name,
                    list(self._transducer_headers.keys()),
                )
                log.warning("Will use ER60 transducer config fields as default")

                transducer_header = self._transducer_headers["ER60"]
                _sounder_name_used = "ER60"

            txcvr_header_fields = [x[0] for x in transducer_header]
            txcvr_header_fmt = "=" + "".join([x[1] for x in transducer_header])
            txcvr_header_size = struct.calcsize(txcvr_header_fmt)  # noqa

            for txcvr_indx, txcvr in list(data["transceivers"].items()):
                txcvr_contents = []

                if _sounder_name_used in ["ER60", "ES60", "ES70"]:
                    for field in txcvr_header_fields[:17]:
                        txcvr_contents.append(txcvr[field])

                    txcvr_contents.extend(txcvr["pulse_length_table"])
                    txcvr_contents.append(txcvr["spare1"])

                    txcvr_contents.extend(txcvr["gain_table"])
                    txcvr_contents.append(txcvr["spare2"])

                    txcvr_contents.extend(txcvr["sa_correction_table"])
                    txcvr_contents.append(txcvr["spare3"])

                    txcvr_contents.extend(
                        [txcvr["gpt_software_version"], txcvr["spare4"]]
                    )

                    txcvr_contents_str = struct.pack(txcvr_header_fmt, *txcvr_contents)

                elif _sounder_name_used == "MBES":
                    for field in txcvr_header_fields:
                        txcvr_contents.append(txcvr[field])

                    txcvr_contents_str = struct.pack(txcvr_header_fmt, *txcvr_contents)

                else:
                    raise RuntimeError(
                        "Unknown _sounder_name_used (Should not happen, this is a bug!)"
                    )

                datagram_fmt += "%ds" % (len(txcvr_contents_str))
                datagram_contents.append(txcvr_contents_str)

        elif version == 1:
            for field in self.header_fields(version):
                datagram_contents.append(data[field])

            datagram_fmt += "%ds" % (len(data["beam_config"]))
            datagram_contents.append(data["beam_config"])

        return struct.pack(datagram_fmt, *datagram_contents)

class SimradRawParser(_SimradDatagramParser):
    """
    Sample Data Datagram parser operates on dictonaries with the following keys:

        type:         string == 'RAW0'
        low_date:     long uint representing LSBytes of 64bit NT date
        high_date:    long uint representing MSBytes of 64bit NT date
        timestamp:    datetime.datetime object of NT date, assumed to be UTC

        channel                         [short] Channel number
        mode                            [short] 1 = Power only, 2 = Angle only 3 = Power & Angle
        transducer_depth                [float]
        frequency                       [float]
        transmit_power                  [float]
        pulse_length                    [float]
        bandwidth                       [float]
        sample_interval                 [float]
        sound_velocity                  [float]
        absorption_coefficient          [float]
        heave                           [float]
        roll                            [float]
        pitch                           [float]
        temperature                     [float]
        heading                         [float]
        transmit_mode                   [short] 0 = Active, 1 = Passive, 2 = Test, -1 = Unknown
        spare0                          [str]
        offset                          [long]
        count                           [long]

        power                           [numpy array] Unconverted power values (if present)
        angle                           [numpy array] Unconverted angle values (if present)

    from_string(str):   parse a raw sample datagram
                        (with leading/trailing datagram size stripped)

    to_string(dict):    Returns raw string (including leading/trailing size fields)
                        ready for writing to disk
    """

    def __init__(self):
        headers = {
            0: [
                ("type", "4s"),
                ("low_date", "L"),
                ("high_date", "L"),
                ("channel", "h"),
                ("mode", "h"),
                ("transducer_depth", "f"),
                ("frequency", "f"),
                ("transmit_power", "f"),
                ("pulse_length", "f"),
                ("bandwidth", "f"),
                ("sample_interval", "f"),
                ("sound_velocity", "f"),
                ("absorption_coefficient", "f"),
                ("heave", "f"),
                ("roll", "f"),
                ("pitch", "f"),
                ("temperature", "f"),
                ("heading", "f"),
                ("transmit_mode", "h"),
                ("spare0", "6s"),
                ("offset", "l"),
                ("count", "l"),
            ],
            3: [
                ("type", "4s"),
                ("low_date", "L"),
                ("high_date", "L"),
                ("channel_id", "128s"),
                ("data_type", "h"),
                ("spare", "2s"),
                ("offset", "l"),
                ("count", "l"),
            ],
        }
        _SimradDatagramParser.__init__(self, "RAW", headers)

    def _unpack_contents(self, raw_string, bytes_read, version):

        header_values = struct.unpack(
            self.header_fmt(version), raw_string[: self.header_size(version)]
        )

        data = {}

        for indx, field in enumerate(self.header_fields(version)):
            data[field] = header_values[indx]
            if isinstance(data[field], bytes):
                data[field] = data[field].decode()

        data["timestamp"] = nt_to_unix((data["low_date"], data["high_date"]))
        data["bytes_read"] = bytes_read

        if version == 0:

            if data["count"] > 0:
                block_size = data["count"] * 2
                indx = self.header_size(version)

                if int(data["mode"]) & 0x1:
                    data["power"] = np.frombuffer(
                        raw_string[indx : indx + block_size], dtype="int16"  # noqa
                    )
                    indx += block_size
                else:
                    data["power"] = None

                if int(data["mode"]) & 0x2:
                    data["angle"] = np.frombuffer(
                        raw_string[indx : indx + block_size], dtype="int8"  # noqa
                    )
                    data["angle"] = data["angle"].reshape((-1, 2))
                else:
                    data["angle"] = None

            else:
                data["power"] = np.empty((0,), dtype="int16")
                data["angle"] = np.empty((0, 2), dtype="int8")

        elif version == 3:

            # result = 1j*Data[...,1]; result += Data[...,0]

            #  clean up the channel ID
            data["channel_id"] = data["channel_id"].strip("\x00")

            if data["count"] > 0:

                #  set the initial block size and indx value.
                block_size = data["count"] * 2
                indx = self.header_size(version)

                if data["data_type"] & 0b1:
                    data["power"] = np.frombuffer(
                        raw_string[indx : indx + block_size], dtype="int16"  # noqa
                    )
                    indx += block_size
                else:
                    data["power"] = None

                if data["data_type"] & 0b10:
                    data["angle"] = np.frombuffer(
                        raw_string[indx : indx + block_size], dtype="int8"  # noqa
                    )
                    data["angle"] = data["angle"].reshape((-1, 2))
                    indx += block_size
                else:
                    data["angle"] = None

                #  determine the complex sample data type - this is contained in bits 2 and 3
                #  of the datatype <short> value. I'm assuming the types are exclusive...
                data["complex_dtype"] = np.float16
                type_bytes = 2
                if data["data_type"] & 0b1000:
                    data["complex_dtype"] = np.float32
                    type_bytes = 8

                #  determine the number of complex samples
                data["n_complex"] = data["data_type"] >> 8

                #  unpack the complex samples
                if data["n_complex"] > 0:
                    #  determine the block size
                    block_size = data["count"] * data["n_complex"] * type_bytes

                    data["complex"] = np.frombuffer(
                        raw_string[indx : indx + block_size],  # noqa
                        dtype=data["complex_dtype"],
                    )
                    data["complex"].dtype = np.complex64
                else:
                    data["complex"] = None

            else:
                data["power"] = np.empty((0,), dtype="int16")
                data["angle"] = np.empty((0,), dtype="int8")
                data["complex"] = np.empty((0,), dtype="complex64")
                data["n_complex"] = 0

        return data

    def _pack_contents(self, data, version):

        datagram_fmt = self.header_fmt(version)

        datagram_contents = []

        if version == 0:

            if data["count"] > 0:
                if (int(data["mode"]) & 0x1) and (
                    len(data.get("power", [])) != data["count"]
                ):
                    log.warning(
                        "Data 'count' = %d, but contains %d power samples.  Ignoring power."
                    )
                    data["mode"] &= ~(1 << 0)

                if (int(data["mode"]) & 0x2) and (
                    len(data.get("angle", [])) != data["count"]
                ):
                    log.warning(
                        "Data 'count' = %d, but contains %d angle samples.  Ignoring angle."
                    )
                    data["mode"] &= ~(1 << 1)

                if data["mode"] == 0:
                    log.warning(
                        "Data 'count' = %d, but mode == 0.  Setting count to 0",
                        data["count"],
                    )
                    data["count"] = 0

            for field in self.header_fields(version):
                datagram_contents.append(data[field])

            if data["count"] > 0:

                if int(data["mode"]) & 0x1:
                    datagram_fmt += "%dh" % (data["count"])
                    datagram_contents.extend(data["power"])

                if int(data["mode"]) & 0x2:
                    datagram_fmt += "%dH" % (data["count"])
                    datagram_contents.extend(data["angle"])

        return struct.pack(datagram_fmt, *datagram_contents)


import os
import tkinter as tk
from tkinter import ttk
from tkinter import filedialog
from tkinter import PhotoImage
from tkinter import messagebox

import pafy
import youtube_dl

# if you get api limit exceeded error, get an api key and paste
# here as a string value
# pafy.set_api_key(key)

# sample video url
# https://www.youtube.com/watch?v=CjeYOtL6ORE

cwd = os.getcwd()

class CustomEntry(tk.Entry):
	def __init__(self, parent, *args, **kwargs):
		tk.Entry.__init__(self, parent, *args, **kwargs)
		self.parent = parent
		self.bind('<FocusOut>', self.add_placeholder)
		self.bind('<FocusIn>', self.clear_placeholder)

		self.configure(fg="gray70")
		self.insert(0, 'Enter Video URL')

	def add_placeholder(self, event=None):
		if not self.get():
			self.configure(fg="gray70")
			self.insert(0, 'Enter Video URL')

	def clear_placeholder(self, event):
		if event and self.get() == 'Enter Video URL':
			self.delete('0', 'end')
			self.configure(fg="black")

# Application Class -----------------------------------------------

class Application(tk.Frame):
	def __init__(self, master=None):
		super().__init__(master=master)
		self.master = master
		self.master.focus_set()
		self.pack()

		self.url = ''
		self.video_quality = tk.StringVar()
		self.filesize = 0

		self.is_video_downloading = False
		self.is_audio_downloading = False

		self.draw_title_frame()
		self.draw_main_frame()

		self.bind('<Return>', self.search_video)

	def draw_title_frame(self):
		self.title_frame = tk.Frame(self, bg='red', width=440, height=60)
		self.title_frame.grid(row=0, column=0, columnspan=5, pady=5)
		self.title_frame.grid_propagate(False)
		self.title = tk.Label(self.title_frame, text=' SaveFromYT - Youtube Audio/Video Downloader',
							fg='white', bg='red', font=('Times', 14),
							width=450, height=50, image=youtube_icon, compound=tk.LEFT,
							anchor = 'w')
		self.title.grid(row=0, column=0, padx=5, ipadx=20)

	def draw_main_frame(self):
		self.main_frame = tk.Frame(self, width=440, height=240, highlightthickness=1,
							highlightbackground='red')
		self.main_frame.grid(row=1, column=0, columnspan=5, pady=5, rowspan=3)
		self.main_frame.grid_propagate(False)

		self.entry = CustomEntry(self.main_frame, width=52)
		self.entry.grid(row=0, column=0, columnspan=3, pady=100, padx=(20,10))
		self.entry.bind('<Return>', self.search_video)

		self.search = tk.Button(self.main_frame, image=search_icon,
							fg='white', cursor='hand2', command=self.search_video,
							relief=tk.FLAT)
		self.search.grid(row=0, column=4, pady=100, padx=(30,10))

	def draw_download_frame(self):
		self.main_frame.destroy()

		self.info_frame = tk.Frame(self, width=150, height=173, highlightthickness=1,
							highlightbackground='red')
		self.info_frame.grid(row=1, column=0, columnspan=2)
		self.info_frame.grid_propagate(False)

		self.video_frame = tk.Frame(self, width=290, height=173, highlightthickness=1,
							highlightbackground='red')
		self.video_frame.grid(row=1, column=2, columnspan=3)
		self.video_frame.grid_propagate(False)

		self.audio_frame = tk.Frame(self, width=370, height=67, highlightthickness=1,
							highlightbackground='red')
		self.audio_frame.grid(row=2, column=0, columnspan=4)
		self.audio_frame.grid_propagate(False)

		self.back_frame = tk.Frame(self, width=70, height=67, highlightthickness=1,
							highlightbackground='red')
		self.back_frame.grid(row=2, column=4)
		self.back_frame.grid_propagate(False)

	def draw_download_widgets(self):
		# self.info_frame

		self.title = tk.Label(self.info_frame, width=20, height=3, bg='red',
					wraplength=120, fg='white')
		self.title.grid(row=0, column=0, padx=1, pady=2)

		self.views = tk.Label(self.info_frame, width=20, height=2, bg='red',
					fg='white')
		self.views.grid(row=1, column=0, padx=1, pady=1)

		self.duration = tk.Label(self.info_frame, width=20, height=2, bg='red',
					fg='white')
		self.duration.grid(row=2, column=0, padx=1, pady=1)

		self.published = tk.Label(self.info_frame, width=20, height=2, bg='red',
					fg='white')
		self.published.grid(row=3, column=0, padx=1, pady=1)

		# self.video_frame

		self.video_quality.set(self.option_streams[0])
		self.options = tk.OptionMenu(self.video_frame, self.video_quality,
						*self.option_streams)
		self.options.config(bg='red', fg='white')
		self.options['menu'].config(bg='red', fg='white')
		self.options.grid(row=0, column=0, padx=50, pady=20, columnspan=5)

		self.video_dwn = tk.Button(self.video_frame, text='Download MP4',
							command=self.download_video, bg='red', fg='white',
							width=15, cursor='hand2')
		self.video_dwn.grid(row=1, column=0, padx=50, pady=10, columnspan=5)

		# self.audio_frame

		self.audio_dwn = tk.Button(self.audio_frame, text='Download MP3',
							command=self.download_mp3, bg='red', fg='white',
							width=15, cursor='hand2')
		self.audio_dwn.grid(row=0, column=0, padx=20, pady=20)

		# self.back_frame
		self.back = tk.Button(self.back_frame, text='back', image=back_icon,
							command=self.go_back, relief=tk.FLAT)
		self.back.grid(row=0, column=0, pady=10, padx=10)

	def cease_buttons(self):
		if self.is_video_downloading:
			self.video_dwn['text'] = 'downloading'
		if self.is_audio_downloading:
			self.audio_dwn['text'] = 'downloading'
		self.video_dwn.config(state='disabled')
		self.audio_dwn.config(state='disabled')

	def release_buttons(self):
		self.video_dwn.config(state='normal')
		self.audio_dwn.config(state='normal')
		if not self.is_video_downloading:
			self.video_dwn['text'] = 'Download MP4'
		if not self.is_audio_downloading:
			self.audio_dwn['text'] = 'Download MP3'

	def search_video(self, event=None):
		self.url = self.entry.get()
		self.master.focus_set()

		if self.url and ' ' not in self.url:
			try:
				video = pafy.new(self.url)
				self.video_title = video.title
				duration = video.duration
				views = video.viewcount
				published = video.published
				thumbnail = video.thumb
				self.streams = video.streams
				self.option_streams = self.streams[::-1]

				self.draw_download_frame()
				self.draw_download_widgets()

				self.title['text'] = self.video_title[:50]
				self.views['text'] = f'Views : {views:,}'
				self.duration['text'] = f'Length : {duration}'
				self.published['text'] = f'Pub : {published[:10]}'
			except OSError:
				messagebox.showerror('SaveFromYT', 'Cannot extract data')
			except ValueError:
				messagebox.showerror('SaveFromYT', 'Invalid URL')
			except:
				messagebox.showerror('SaveFromYT', 'Cannot connect with internet')

	def download_video(self):
		filetypes = [('MP4', '.mp4')]
		filepath = filedialog.asksaveasfilename(initialdir=cwd,
						initialfile=self.video_title[:25]+'.mp4',
						filetypes=filetypes)
		if filepath:
			self.is_video_downloading = True
			self.cease_buttons()
			vq = self.video_quality.get()
			l = len(self.streams)
			opts = [str(stream) for stream in self.option_streams]
			stream = self.streams[opts.index(vq) - l + 1]
			self.filesize = stream.get_filesize()

			self.sizelabel = tk.Label(self.video_frame, bg='red', fg='white',
							text=f'Filesize : {self.filesize/(1024*1024):.2f} Mb')
			self.sizelabel.grid(row=2, column=0, pady=5)
			self.pb = ttk.Progressbar(self.video_frame, orient=tk.HORIZONTAL,
							mode='determinate', length=100)
			self.pb.grid(row=2, column=2, columnspan=3, pady=5)

			try:
				stream.download(quiet=True, callback=self.download_callback,
								filepath=filepath)
				messagebox.showinfo('SaveFromYT', 'Video Downloaded Successfully')
			except:
				messagebox.showerror('SaveFromYT', 'Cannot connect with internet')

			self.pb.destroy()
			self.sizelabel.destroy()
			self.is_video_downloading = False
			self.release_buttons()

	def download_callback(self, total, recvd, ratio, rate, eta):
		perc = (recvd / total) * 100
		self.pb['value'] = int(perc)
		self.update()

	def download_mp3(self):
		filetypes = ['MP3', '.mp3']
		filepath = filedialog.asksaveasfilename(initialdir=cwd,
						initialfile=''.join(self.video_title[:25]+'.mp3'))
		if filepath:
			ydl_opts = {
				'format': 'bestaudio/best',
				'outtmpl' : filepath,
				'postprocessors': [{
					'key': 'FFmpegExtractAudio',
					'preferredcodec': 'mp3',
					'preferredquality': '192'
				}],
				'postprocessor_args': [
					'-ar', '16000'
				],
				'prefer_ffmpeg': True,
				'keepvideo': True,
				'progress_hooks': [self.download_hook]
			}

			self.is_audio_downloading = True
			self.cease_buttons()

			try:
				self.pb = ttk.Progressbar(self.audio_frame, orient=tk.HORIZONTAL,
							mode='determinate', length=100)
				self.pb.grid(row=0, column=2, pady=20, padx=20)

				with youtube_dl.YoutubeDL(ydl_opts) as ydl:
					ydl.download([self.url])

				for file in os.listdir():
					if file.endswith('.webm'):
						os.remove(file)

				self.pb.destroy()
				messagebox.showinfo('SaveFromYT', 'Successfully Downloaded Mp3')
			except:
				messagebox.showinfo('SaveFromYT', "Can't connect with internet")

			self.is_audio_downloading = False
			self.release_buttons()

	def download_hook(self, d):
		if d['status'] == 'downloading':
			p = d['_percent_str']
			p = float(p.replace('%','').replace(' ',''))
			self.pb['value'] = round(p)
			self.update()

	def go_back(self):
		self.info_frame.destroy()
		self.video_frame.destroy()
		self.audio_frame.destroy()
		self.back_frame.destroy()

		self.draw_main_frame()

if __name__ == '__main__':
	root = tk.Tk()
	root.geometry('450x320')
	root.title('SaveFromYT')
	root.resizable(0,0)

	youtube_icon = PhotoImage(file='icons/youtube.png')
	back_icon = PhotoImage(file='icons/back.png')
	search_icon = PhotoImage(file='icons/search.png')

	app = Application(master=root)
	app.mainloop()


#!/usr/bin/env python
import glob
import os
import sys
import time
import re
import shutil
from os.path import expanduser
home = expanduser("~")
rosey_dir = home + "/.rosey/"
rosey_config = rosey_dir + "config"
rosey_log = rosey_dir + "rosey.log"

class Rosey():

  def __init__(self, config):
    """Configs come one in a list of three member lists"""
    """glob-file-pattern-to-match, prefix to remove, directory-to-move-matched-files"""
    self.config = config

  def FileMoveToDoList(self):
    files = []
    todos = []
    configs = iter(self.config)
    for config in configs:
      files = self.findMatchingFiles(config[0])
      for f in files:
        todos +=  [
          [f, self.replacePatternWithNewPath(f, config[1], config[2])]
        ]

    return todos

  def replacePatternWithNewPath(self, file, remove_this, dest_path):
    t = time.localtime(os.path.getctime(file))
    timestamp = time.strftime("%Y-%m-%d", t) + "-"
    orig_name = os.path.basename(file)
    trimmed_name = orig_name.replace(remove_this, "")
    no_spaces_name = trimmed_name.replace(" ", "-")
    timestamped_name = timestamp + no_spaces_name
    new_name = re.sub("-+", "-", timestamped_name)
    new_path = dest_path + new_name
    return new_path

  def findMatchingFiles(self, glob_spec):
    all = glob.glob(glob_spec)
    return all

def check_config(config):
  findings = []
  regexes = [f[0] for f in config]
  if (regexes.sort() != list(set(regexes)).sort()):
    findings += "You have one or more duplicate patterns"
    return

  dest_dirs = [f[2] for f in config]
  for dest in dest_dirs:
    if (not os.path.isdir(dest)):
      findings += "Destination directory does not exist: '{0}'".format(dest)

  return findings

def cleanup_config(config):
  config_list = [line.rstrip().split(',') for line in config]
  trimmed_config = []
  for config_item in config_list:
    trimmed_config += [[f.lstrip().rstrip() for f in config_item]]

  return trimmed_config

def moveEm(todo, really_move = True):

  with open(rosey_log, "a") as myfile:
      for t in todo:
        message = "Moving: {0}\n    to: {1}\n".format(t[0], t[1])

        if really_move:
          try:
            shutil.move(t[0], t[1])
            message += "      : Move Successful"
          except Exception as e:
            message += "      : Move Fails {0}.".format(e)

          myfile.write(message + "\n");
        print message

def show_findings(findings):
  for f in findings:
    print f

def main(arg):

  if (not os.path.exists(rosey_config)):
    print "You need to create ~/.rosey/config"
    exit(1)

  with open(rosey_config) as f:
    config = f.readlines()

  clean_config = cleanup_config(config)
  findings = check_config(clean_config)
  if findings != []:
    show_findings(findings)
    exit(1)
  rosey = Rosey(clean_config)
  todo = rosey.FileMoveToDoList()
  if arg == "move":
    moveEm(todo)
  if arg == "show":
    moveEm(todo, False)

if __name__ == '__main__':
  if len(sys.argv) != 2:
    print "usage: {0} [move, show] {1}".format(sys.argv[0], len(sys.argv))
    exit(0)
  main(sys.argv[1]) # Run the example


"""Test different accessory types: HumidifierDehumidifier."""
from pyhap.const import (
    CATEGORY_HUMIDIFIER,
    HAP_REPR_AID,
    HAP_REPR_CHARS,
    HAP_REPR_IID,
    HAP_REPR_VALUE,
)

from homeassistant.components.homekit.const import (
    ATTR_VALUE,
    CONF_LINKED_HUMIDITY_SENSOR,
    PROP_MAX_VALUE,
    PROP_MIN_STEP,
    PROP_MIN_VALUE,
    PROP_VALID_VALUES,
)
from homeassistant.components.homekit.type_humidifiers import HumidifierDehumidifier
from homeassistant.components.humidifier.const import (
    ATTR_HUMIDITY,
    ATTR_MAX_HUMIDITY,
    ATTR_MIN_HUMIDITY,
    DEFAULT_MAX_HUMIDITY,
    DEFAULT_MIN_HUMIDITY,
    DEVICE_CLASS_DEHUMIDIFIER,
    DEVICE_CLASS_HUMIDIFIER,
    DOMAIN,
    SERVICE_SET_HUMIDITY,
)
from homeassistant.const import (
    ATTR_DEVICE_CLASS,
    ATTR_ENTITY_ID,
    ATTR_UNIT_OF_MEASUREMENT,
    DEVICE_CLASS_HUMIDITY,
    PERCENTAGE,
    SERVICE_TURN_OFF,
    SERVICE_TURN_ON,
    STATE_OFF,
    STATE_ON,
    STATE_UNAVAILABLE,
)

from tests.common import async_mock_service

async def test_humidifier(hass, hk_driver, events):
    """Test if humidifier accessory and HA are updated accordingly."""
    entity_id = "humidifier.test"

    hass.states.async_set(entity_id, STATE_OFF)
    await hass.async_block_till_done()
    acc = HumidifierDehumidifier(
        hass, hk_driver, "HumidifierDehumidifier", entity_id, 1, None
    )
    hk_driver.add_accessory(acc)

    await acc.run()
    await hass.async_block_till_done()

    assert acc.aid == 1
    assert acc.category == CATEGORY_HUMIDIFIER

    assert acc.char_current_humidifier_dehumidifier.value == 0
    assert acc.char_target_humidifier_dehumidifier.value == 1
    assert acc.char_current_humidity.value == 0
    assert acc.char_target_humidity.value == 45.0
    assert acc.char_active.value == 0

    assert acc.char_target_humidity.properties[PROP_MAX_VALUE] == DEFAULT_MAX_HUMIDITY
    assert acc.char_target_humidity.properties[PROP_MIN_VALUE] == DEFAULT_MIN_HUMIDITY
    assert acc.char_target_humidity.properties[PROP_MIN_STEP] == 1.0
    assert acc.char_target_humidifier_dehumidifier.properties[PROP_VALID_VALUES] == {
        "Humidifier": 1
    }

    hass.states.async_set(
        entity_id,
        STATE_ON,
        {ATTR_HUMIDITY: 47},
    )
    await hass.async_block_till_done()
    assert acc.char_target_humidity.value == 47.0
    assert acc.char_current_humidifier_dehumidifier.value == 2
    assert acc.char_target_humidifier_dehumidifier.value == 1
    assert acc.char_active.value == 1

    hass.states.async_set(
        entity_id,
        STATE_OFF,
        {ATTR_HUMIDITY: 42, ATTR_DEVICE_CLASS: DEVICE_CLASS_HUMIDIFIER},
    )
    await hass.async_block_till_done()
    assert acc.char_target_humidity.value == 42.0
    assert acc.char_current_humidifier_dehumidifier.value == 0
    assert acc.char_target_humidifier_dehumidifier.value == 1
    assert acc.char_active.value == 0

    # Set from HomeKit
    call_set_humidity = async_mock_service(hass, DOMAIN, SERVICE_SET_HUMIDITY)

    char_target_humidity_iid = acc.char_target_humidity.to_HAP()[HAP_REPR_IID]

    hk_driver.set_characteristics(
        {
            HAP_REPR_CHARS: [
                {
                    HAP_REPR_AID: acc.aid,
                    HAP_REPR_IID: char_target_humidity_iid,
                    HAP_REPR_VALUE: 39.0,
                },
            ]
        },
        "mock_addr",
    )

    await hass.async_block_till_done()
    assert len(call_set_humidity) == 1
    assert call_set_humidity[0].data[ATTR_ENTITY_ID] == entity_id
    assert call_set_humidity[0].data[ATTR_HUMIDITY] == 39.0
    assert acc.char_target_humidity.value == 39.0
    assert len(events) == 1
    assert events[-1].data[ATTR_VALUE] == "RelativeHumidityHumidifierThreshold to 39.0%"

async def test_dehumidifier(hass, hk_driver, events):
    """Test if dehumidifier accessory and HA are updated accordingly."""
    entity_id = "humidifier.test"

    hass.states.async_set(
        entity_id, STATE_OFF, {ATTR_DEVICE_CLASS: DEVICE_CLASS_DEHUMIDIFIER}
    )
    await hass.async_block_till_done()
    acc = HumidifierDehumidifier(
        hass, hk_driver, "HumidifierDehumidifier", entity_id, 1, None
    )
    hk_driver.add_accessory(acc)

    await acc.run()
    await hass.async_block_till_done()

    assert acc.aid == 1
    assert acc.category == CATEGORY_HUMIDIFIER

    assert acc.char_current_humidifier_dehumidifier.value == 0
    assert acc.char_target_humidifier_dehumidifier.value == 2
    assert acc.char_current_humidity.value == 0
    assert acc.char_target_humidity.value == 45.0
    assert acc.char_active.value == 0

    assert acc.char_target_humidity.properties[PROP_MAX_VALUE] == DEFAULT_MAX_HUMIDITY
    assert acc.char_target_humidity.properties[PROP_MIN_VALUE] == DEFAULT_MIN_HUMIDITY
    assert acc.char_target_humidity.properties[PROP_MIN_STEP] == 1.0
    assert acc.char_target_humidifier_dehumidifier.properties[PROP_VALID_VALUES] == {
        "Dehumidifier": 2
    }

    hass.states.async_set(
        entity_id,
        STATE_ON,
        {ATTR_HUMIDITY: 30},
    )
    await hass.async_block_till_done()
    assert acc.char_target_humidity.value == 30.0
    assert acc.char_current_humidifier_dehumidifier.value == 3
    assert acc.char_target_humidifier_dehumidifier.value == 2
    assert acc.char_active.value == 1

    hass.states.async_set(
        entity_id,
        STATE_OFF,
        {ATTR_HUMIDITY: 42},
    )
    await hass.async_block_till_done()
    assert acc.char_target_humidity.value == 42.0
    assert acc.char_current_humidifier_dehumidifier.value == 0
    assert acc.char_target_humidifier_dehumidifier.value == 2
    assert acc.char_active.value == 0

    # Set from HomeKit
    call_set_humidity = async_mock_service(hass, DOMAIN, SERVICE_SET_HUMIDITY)

    char_target_humidity_iid = acc.char_target_humidity.to_HAP()[HAP_REPR_IID]

    hk_driver.set_characteristics(
        {
            HAP_REPR_CHARS: [
                {
                    HAP_REPR_AID: acc.aid,
                    HAP_REPR_IID: char_target_humidity_iid,
                    HAP_REPR_VALUE: 39.0,
                },
            ]
        },
        "mock_addr",
    )

    await hass.async_block_till_done()
    assert len(call_set_humidity) == 1
    assert call_set_humidity[0].data[ATTR_ENTITY_ID] == entity_id
    assert call_set_humidity[0].data[ATTR_HUMIDITY] == 39.0
    assert acc.char_target_humidity.value == 39.0
    assert len(events) == 1
    assert (
        events[-1].data[ATTR_VALUE] == "RelativeHumidityDehumidifierThreshold to 39.0%"
    )

async def test_hygrostat_power_state(hass, hk_driver, events):
    """Test if accessory and HA are updated accordingly."""
    entity_id = "humidifier.test"

    hass.states.async_set(
        entity_id,
        STATE_ON,
        {ATTR_HUMIDITY: 43},
    )
    await hass.async_block_till_done()
    acc = HumidifierDehumidifier(
        hass, hk_driver, "HumidifierDehumidifier", entity_id, 1, None
    )
    hk_driver.add_accessory(acc)

    await acc.run()
    await hass.async_block_till_done()

    assert acc.char_current_humidifier_dehumidifier.value == 2
    assert acc.char_target_humidifier_dehumidifier.value == 1
    assert acc.char_active.value == 1

    hass.states.async_set(
        entity_id,
        STATE_OFF,
        {ATTR_HUMIDITY: 43},
    )
    await hass.async_block_till_done()
    assert acc.char_current_humidifier_dehumidifier.value == 0
    assert acc.char_target_humidifier_dehumidifier.value == 1
    assert acc.char_active.value == 0

    # Set from HomeKit
    call_turn_on = async_mock_service(hass, DOMAIN, SERVICE_TURN_ON)

    char_active_iid = acc.char_active.to_HAP()[HAP_REPR_IID]

    hk_driver.set_characteristics(
        {
            HAP_REPR_CHARS: [
                {
                    HAP_REPR_AID: acc.aid,
                    HAP_REPR_IID: char_active_iid,
                    HAP_REPR_VALUE: 1,
                },
            ]
        },
        "mock_addr",
    )

    await hass.async_block_till_done()
    assert len(call_turn_on) == 1
    assert call_turn_on[0].data[ATTR_ENTITY_ID] == entity_id
    assert acc.char_active.value == 1
    assert len(events) == 1
    assert events[-1].data[ATTR_VALUE] == "Active to 1"

    call_turn_off = async_mock_service(hass, DOMAIN, SERVICE_TURN_OFF)

    hk_driver.set_characteristics(
        {
            HAP_REPR_CHARS: [
                {
                    HAP_REPR_AID: acc.aid,
                    HAP_REPR_IID: char_active_iid,
                    HAP_REPR_VALUE: 0,
                },
            ]
        },
        "mock_addr",
    )

    await hass.async_block_till_done()
    assert len(call_turn_off) == 1
    assert call_turn_off[0].data[ATTR_ENTITY_ID] == entity_id
    assert acc.char_active.value == 0
    assert len(events) == 2
    assert events[-1].data[ATTR_VALUE] == "Active to 0"

async def test_hygrostat_get_humidity_range(hass, hk_driver):
    """Test if humidity range is evaluated correctly."""
    entity_id = "humidifier.test"

    hass.states.async_set(
        entity_id, STATE_OFF, {ATTR_MIN_HUMIDITY: 40, ATTR_MAX_HUMIDITY: 45}
    )
    await hass.async_block_till_done()
    acc = HumidifierDehumidifier(
        hass, hk_driver, "HumidifierDehumidifier", entity_id, 1, None
    )
    hk_driver.add_accessory(acc)

    await acc.run()
    await hass.async_block_till_done()

    assert acc.char_target_humidity.properties[PROP_MAX_VALUE] == 45
    assert acc.char_target_humidity.properties[PROP_MIN_VALUE] == 40

async def test_humidifier_with_linked_humidity_sensor(hass, hk_driver):
    """Test a humidifier with a linked humidity sensor can update."""
    humidity_sensor_entity_id = "sensor.bedroom_humidity"

    hass.states.async_set(
        humidity_sensor_entity_id,
        "42.0",
        {
            ATTR_DEVICE_CLASS: DEVICE_CLASS_HUMIDITY,
            ATTR_UNIT_OF_MEASUREMENT: PERCENTAGE,
        },
    )
    await hass.async_block_till_done()
    entity_id = "humidifier.test"

    hass.states.async_set(entity_id, STATE_OFF)
    await hass.async_block_till_done()
    acc = HumidifierDehumidifier(
        hass,
        hk_driver,
        "HumidifierDehumidifier",
        entity_id,
        1,
        {CONF_LINKED_HUMIDITY_SENSOR: humidity_sensor_entity_id},
    )
    hk_driver.add_accessory(acc)

    await acc.run()
    await hass.async_block_till_done()

    assert acc.char_current_humidity.value == 42.0

    hass.states.async_set(
        humidity_sensor_entity_id,
        "43.0",
        {
            ATTR_DEVICE_CLASS: DEVICE_CLASS_HUMIDITY,
            ATTR_UNIT_OF_MEASUREMENT: PERCENTAGE,
        },
    )
    await hass.async_block_till_done()

    assert acc.char_current_humidity.value == 43.0

    hass.states.async_set(
        humidity_sensor_entity_id,
        STATE_UNAVAILABLE,
        {
            ATTR_DEVICE_CLASS: DEVICE_CLASS_HUMIDITY,
            ATTR_UNIT_OF_MEASUREMENT: PERCENTAGE,
        },
    )
    await hass.async_block_till_done()

    assert acc.char_current_humidity.value == 43.0

    hass.states.async_remove(humidity_sensor_entity_id)
    await hass.async_block_till_done()

    assert acc.char_current_humidity.value == 43.0

async def test_humidifier_with_a_missing_linked_humidity_sensor(hass, hk_driver):
    """Test a humidifier with a configured linked motion sensor that is missing."""
    humidity_sensor_entity_id = "sensor.bedroom_humidity"
    entity_id = "humidifier.test"

    hass.states.async_set(entity_id, STATE_OFF)
    await hass.async_block_till_done()
    acc = HumidifierDehumidifier(
        hass,
        hk_driver,
        "HumidifierDehumidifier",
        entity_id,
        1,
        {CONF_LINKED_HUMIDITY_SENSOR: humidity_sensor_entity_id},
    )
    hk_driver.add_accessory(acc)

    await acc.run()
    await hass.async_block_till_done()

    assert acc.char_current_humidity.value == 0

async def test_humidifier_as_dehumidifier(hass, hk_driver, events, caplog):
    """Test an invalid char_target_humidifier_dehumidifier from HomeKit."""
    entity_id = "humidifier.test"

    hass.states.async_set(entity_id, STATE_OFF)
    await hass.async_block_till_done()
    acc = HumidifierDehumidifier(
        hass, hk_driver, "HumidifierDehumidifier", entity_id, 1, None
    )
    hk_driver.add_accessory(acc)

    await acc.run()
    await hass.async_block_till_done()

    assert acc.char_target_humidifier_dehumidifier.value == 1

    # Set from HomeKit
    char_target_humidifier_dehumidifier_iid = (
        acc.char_target_humidifier_dehumidifier.to_HAP()[HAP_REPR_IID]
    )

    hk_driver.set_characteristics(
        {
            HAP_REPR_CHARS: [
                {
                    HAP_REPR_AID: acc.aid,
                    HAP_REPR_IID: char_target_humidifier_dehumidifier_iid,
                    HAP_REPR_VALUE: 0,
                },
            ]
        },
        "mock_addr",
    )

    await hass.async_block_till_done()
    assert "TargetHumidifierDehumidifierState is not supported" in caplog.text
    assert len(events) == 0


#!/usr/bin/python
# -*- coding: utf8 -*-

import os
import logging

import sys
import argparse
sys.path.append("../core")
from qgis_project_substitute import substitute_project

from processor import Processor

def argparser_prepare():

    class PrettyFormatter(argparse.ArgumentDefaultsHelpFormatter,
        argparse.RawDescriptionHelpFormatter):

        max_help_position = 35

    parser = argparse.ArgumentParser(description='OSMTram process',
            formatter_class=PrettyFormatter)
    parser.add_argument('--prune',dest='prune', required=False, action='store_true', help='Clear temporary folder')
    parser.add_argument('--skip-osmupdate',dest='skip_osmupdate', required=False, action='store_true')
    parser.add_argument('--workdir',dest='WORKDIR', required=True)

    parser.epilog = \
        '''Samples:
%(prog)s

''' \
        % {'prog': parser.prog}
    return parser

dump_url = 'http://download.geofabrik.de/europe/latvia-latest.osm.pbf'

parser = argparser_prepare()
args = parser.parse_args()

WORKDIR=args.WORKDIR

logging.basicConfig(level=logging.DEBUG,format='%(asctime)s %(levelname)-8s %(message)s',datefmt='%Y-%m-%d %H:%M:%S')
logger = logging.getLogger(__name__)

logger.info('Start')

processor = Processor()

processor.process_sheets('latvia.geojson',WORKDIR,dump_url,dump_name='latvia')

#,attribute_filter='''"name_ru"= 'Лиепая' and "type"='tram' '''


import numpy as np

from src.util import Util, Article

class Answer:
    """Answer questions based on the initialized article."""

    def __init__(self, article):
        """
        Create a new instance of the Answer class.

        Args:
            article: An instance of the Article class
        """
        self.article = article

    def answer(self, question, return_score=False):
        """
        Answer the given question.

        Args:
            question: Question string

        Returns:
            Answer to question as string
        """
        u = Util()
        question_embedding = u.embeddings([question])[0]

        sentences_list = []

        for paragraph in self.article.sentences:
            paragraph_text = [s.text for s in paragraph]
            sentences_list += paragraph_text

        sentences_embeddings = u.embeddings(sentences_list)

        distances = []
        for i, embedding in enumerate(sentences_embeddings):
            diffs = np.inner(question_embedding, embedding)
            dist = diffs

            distances.append((dist, sentences_list[i]))

        distances.sort(key=lambda x: x[0], reverse=True)

        most_similar_sentence = distances[0][1]
        most_similar_score = distances[0][0]

        if return_score:
            return (most_similar_sentence, most_similar_score)

        return most_similar_sentence

if __name__ == "__main__":
    u = Util()
    art = Article(u.load_txt_article("../articles/Development_data/set4/set4/a1.txt"))
    a = Answer(art)
    q = "Who studied the stars of the southern hemisphere from 1750 until 1754 from Cape of Good Hope?"
    print(a.answer(q))

# Who is a product of a revision of the Old Babylonian system in later Neo-Babylonian astronomy 6th century BC?
# Who interpreted the creatures appearing in the books of Ezekiel (and thence in Revelation) as the middle signs of the four quarters of the Zodiac?
# Who studied the stars of the southern hemisphere from 1750 until 1754 from Cape of Good Hope?
# Who aided the IAU (International Astronomical Union) in dividing the celestial sphere into 88 official constellations?
# Who is a product of a revision of the Old Babylonian system in later Neo-Babylonian astronomy 6th century BC?