pybufrkit/templatedata.py
class TemplateData(object):
    """
    This class is dedicated to the data section of a BUFR message and produces a
    fully hierarchical structure for the otherwise flat list of decoded
    descriptors and values. Attributes like associated fields and statistical
    values are properly allocated to their corresponding referred elements.
    """

    def __init__(self,
                 template,
                 is_compressed,
                 decoded_descriptors_all_subsets,
                 decoded_values_all_subsets,
                 bitmap_links_all_subsets):

        self.template = template
        self.is_compressed = is_compressed
        self.decoded_descriptors_all_subsets = decoded_descriptors_all_subsets
        self.decoded_values_all_subsets = decoded_values_all_subsets
        self.bitmap_links_all_subsets = bitmap_links_all_subsets
        self.n_subsets = len(self.decoded_descriptors_all_subsets)

        # The most important difference between decoded nodes and decoded
        # descriptors is that the decoded descriptors are flat while decoded
        # nodes are nested. For an example, the replication is a single node
        # (with its own member nodes) in the decoded nodes but a series of flat
        # descriptors in decoded descriptors.

        if self.is_compressed:
            self.decoded_nodes_all_subsets = [[]] * self.n_subsets
        else:
            self.decoded_nodes_all_subsets = [[] for _ in range(self.n_subsets)]

        self.decoded_nodes = [] if not self.decoded_nodes_all_subsets else self.decoded_nodes_all_subsets[0]

        self._is_wired = False

    def wire(self):
        """
        From the flat list of descriptors and values, construct a fully
        hierarchical structure of data including sequence descriptors and
        correctly set bitmapped values to their corresponding node as
        attributes.
        """
        # Do not wire more than once
        if self._is_wired:
            return
        else:
            self._is_wired = True

        # For compressed data, the wiring is the same for all subsets.
        n_subsets = 1 if self.is_compressed else self.n_subsets

        for idx_subset in range(n_subsets):
            self.decoded_nodes = self.decoded_nodes_all_subsets[idx_subset]
            self.decoded_descriptors = self.decoded_descriptors_all_subsets[idx_subset]
            self.decoded_values = self.decoded_values_all_subsets[idx_subset]
            self.bitmap_links = self.bitmap_links_all_subsets[idx_subset]

            # The index is used to index into the decoded descriptors/values.
            # The is used to have links between flat indices and nested nodes,
            # so that attributes can be associated to their bit-mapped nodes.
            self.index_to_node = {}
            self.next_index = functools.partial(next, itertools.count())
            self.nbits_associated_list = []  # 204 YYY
            self.data_not_present_count = 0  # 221
            self.waiting_for_qa_info_meaning = False
            self.waiting_for_1st_order_stats_meaning = False
            self.waiting_for_difference_stats_meaning = False

            self.wire_members(self.template.members)

            # release memory
            del self.index_to_node

    def get_next_descriptor_and_index(self):
        index = self.next_index()
        return self.decoded_descriptors[index], index

    def add_node(self, node):
        self.decoded_nodes.append(node)
        if not isinstance(node, NoValueDataNode):
            self.index_to_node[node.index] = node
        return node

    def add_value_node(self):
        node = ValueDataNode(*self.get_next_descriptor_and_index())
        self.decoded_nodes.append(node)
        self.index_to_node[node.index] = node
        return node

    def add_delayed_replication_factor_node(self):
        factor_node = ValueDataNode(*self.get_next_descriptor_and_index())
        self.index_to_node[factor_node.index] = factor_node
        return factor_node

    def wire_element_descriptor(self, descriptor):
        # Read associated field if exists
        if self.nbits_associated_list and descriptor.X != 31:
            assoc_node = AssociatedFieldNode(*self.get_next_descriptor_and_index())
            assoc_node.add_attribute(self.associated_field_meaning)
            node = ValueDataNode(*self.get_next_descriptor_and_index())
            node.add_attribute(assoc_node)
            self.add_node(node)

        else:
            if descriptor.X == 33 and self.waiting_for_qa_info_meaning:
                node = self.add_node(QualityInfoNode(*self.get_next_descriptor_and_index()))
                self.index_to_node[self.bitmap_links[node.index]].add_attribute(node)

            else:
                node = self.add_value_node()
                if descriptor.id == 31021 and self.nbits_associated_list:
                    self.associated_field_meaning = node

                elif descriptor.id == 8023 and self.waiting_for_1st_order_stats_meaning:
                    self.first_order_stats_meaning = node
                    self.waiting_for_1st_order_stats_meaning = False

                elif descriptor.id == 8024 and self.waiting_for_difference_stats_meaning:
                    self.difference_stats_meaning = node
                    self.waiting_for_difference_stats_meaning = False

    def wire_fixed_replication_descriptor(self, descriptor):
        """
        :param FixedReplicationDescriptor descriptor:
        """
        fixed_replication_node = FixedReplicationNode(descriptor)
        nodes = self.decoded_nodes
        self.decoded_nodes = fixed_replication_node.members
        for _ in range(descriptor.n_repeats):
            self.wire_members(descriptor.members)
        self.decoded_nodes = nodes

        self.add_node(fixed_replication_node)

    def wire_delayed_replication_descriptor(self, descriptor):
        """
        :param DelayedReplicationDescriptor descriptor:
        """
        delayed_replication_node = DelayedReplicationNode(descriptor)
        nodes = self.decoded_nodes
        self.decoded_nodes = delayed_replication_node.members

        # Add the delayed replication factor node to the node indices as well as
        # it is possible to have attributes attached to it. For an example,
        # ocea_133.bufr from benchmark data has QA info attached to 031001.
        factor_node = self.add_delayed_replication_factor_node()
        delayed_replication_node.factor = factor_node
        for _ in range(self.decoded_values[factor_node.index]):
            self.wire_members(descriptor.members)
        self.decoded_nodes = nodes

        self.add_node(delayed_replication_node)

    def wire_sequence_descriptor(self, descriptor):
        sequence_node = SequenceNode(descriptor)
        nodes = self.decoded_nodes
        self.decoded_nodes = sequence_node.members
        self.wire_members(descriptor.members)
        self.decoded_nodes = nodes

        self.add_node(sequence_node)

    def wire_bitmap_attribute(self, attr_node):
        self.index_to_node[self.bitmap_links[attr_node.index]].add_attribute(attr_node)

    def wire_operator_descriptor(self, descriptor):
        """
        :param OperatorDescriptor descriptor:
        :return:
        """
        operator_code, operand_value = descriptor.operator_code, descriptor.operand_value

        if operator_code in (201, 202, 203, 206, 207, 208,):
            # nbits offset, scale offset, new refval, skip local, increment, change string length
            self.add_node(NoValueDataNode(descriptor))

        elif operator_code == 204:  # associated field
            if operand_value == 0:
                self.nbits_associated_list.pop()
            else:
                self.nbits_associated_list.append(operand_value)
            self.add_node(NoValueDataNode(descriptor))

        elif operator_code == 205:  # read string of YYY bytes
            self.add_value_node()

        # Data not present for following YYY descriptors except class 0-9 and 31
        elif operator_code == 221:
            self.data_not_present_count = operand_value
            self.add_node(NoValueDataNode(descriptor))

        elif operator_code == 222:  # quality info follows
            self.waiting_for_qa_info_meaning = True
            self.add_value_node()

        elif operator_code == 223:  # substituted value
            self.waiting_for_qa_info_meaning = False
            if operand_value == 0:
                self.add_value_node()
            else:
                node = self.add_node(SubstitutionNode(*self.get_next_descriptor_and_index()))
                self.wire_bitmap_attribute(node)

        elif operator_code == 224:  # 1st order stats
            self.waiting_for_qa_info_meaning = False
            if operand_value == 0:
                self.waiting_for_1st_order_stats_meaning = True
                self.add_value_node()
            else:
                node = self.add_node(FirstOrderStatsNode(*self.get_next_descriptor_and_index()))
                node.add_attribute(self.first_order_stats_meaning)
                self.wire_bitmap_attribute(node)

        elif operator_code == 225:  # difference stats
            self.waiting_for_qa_info_meaning = False
            if operand_value == 0:
                self.waiting_for_difference_stats_meaning = True
                self.add_value_node()
            else:
                node = self.add_node(DifferenceStatsNode(*self.get_next_descriptor_and_index()))
                node.add_attribute(self.difference_stats_meaning)
                self.wire_bitmap_attribute(node)

        elif operator_code == 232:  # replaced/retained value
            self.waiting_for_qa_info_meaning = False
            if operand_value == 0:
                self.add_value_node()
            else:
                node = self.add_node(ReplacementNode(*self.get_next_descriptor_and_index()))
                self.wire_bitmap_attribute(node)

        elif operator_code == 235:  # cancel all backwards data reference
            self.waiting_for_qa_info_meaning = False
            self.add_node(NoValueDataNode(descriptor))

        elif operator_code == 236:
            self.add_value_node()

        elif operator_code == 237:
            self.add_value_node()

        else:  # TODO: 241, 242, 243
            raise NotImplementedError('Operator Descriptor {} not implemented'.format(descriptor))

    def wire_skippable_local_descriptor(self):
        self.add_value_node()

    def wire_members(self, members):
        for member in members:

            # 221 YYY data not present for following YYY descriptors except class 0-9 and 31
            if self.data_not_present_count:
                self.data_not_present_count -= 1
                if isinstance(member, ElementDescriptor):
                    X = member.X
                    if not (1 <= X <= 9 or X == 31):  # skipping
                        self.add_node(NoValueDataNode(member))
                        continue

            # Now process normally
            if isinstance(member, ElementDescriptor):
                self.wire_element_descriptor(member)

            elif isinstance(member, FixedReplicationDescriptor):
                self.wire_fixed_replication_descriptor(member)

            elif isinstance(member, DelayedReplicationDescriptor):
                self.wire_delayed_replication_descriptor(member)

            elif isinstance(member, OperatorDescriptor):
                self.wire_operator_descriptor(member)

            elif isinstance(member, SequenceDescriptor):
                self.wire_sequence_descriptor(member)

            elif isinstance(member, SkippedLocalDescriptor):
                self.wire_skippable_local_descriptor()

            elif isinstance(member, UndefinedElementDescriptor):
                # TODO: assume any undefined element descriptor here is a skipped local
                self.wire_skippable_local_descriptor()

            else:
                raise PyBufrKitError('Cannot wire descriptor type: {}'.format(type(member)))
pybufrkit/constants.py
NBITS_FOR_NBITS_DIFF = 6
pybufrkit/constants.py
PARAMETER_TYPE_TEMPLATE_DATA = 'template_data'
pybufrkit/constants.py
BITPOS_START = 'bitpos_start'
pybufrkit/coder.py
class CoderState(object):
    """
    The state of Coder for keeping track of variables when a Coder is working. The use
    of a new state for each run makes it possible to use a single Coder to run
    multiple decoding/encoding tasks.

    :param decoded_values_all_subsets: This is only for Encoder use.
    """

    def __init__(self, is_compressed, n_subsets, decoded_values_all_subsets=None):

        self.is_compressed = is_compressed
        self.n_subsets = n_subsets

        self.idx_subset = 0

        # bitmap_links is a dictionary with key being the index to an attribute descriptor,
        # e.g. QA info, Stats, and the value being the index to the descriptor that the
        # attribute should be attached to.

        if is_compressed:
            # Compressed data has exactly the same decoded descriptors for each subset
            # Hence elements in the all_subsets list reference to the same object.
            self.decoded_descriptors_all_subsets = [[]] * n_subsets
            self.bitmap_links_all_subsets = [{}] * n_subsets
        else:
            # For Uncompressed data, each element of all_subsets are independent
            self.decoded_descriptors_all_subsets = [[] for _ in range(n_subsets)]
            self.bitmap_links_all_subsets = [{} for _ in range(n_subsets)]

        # For uncompressed data, the following two values will be changed during
        # subset context switching
        self.decoded_descriptors = [] if n_subsets == 0 else self.decoded_descriptors_all_subsets[0]
        self.bitmap_links = [] if n_subsets == 0 else self.bitmap_links_all_subsets[0]

        # When debug is turned on, use AuditedList for more logging messages.
        # Each element in the values all_subsets is different compressed or not
        if logging.root.level == logging.getLevelName('DEBUG'):
            if decoded_values_all_subsets:
                self.decoded_values_all_subsets = [AuditedList(vals) for vals in decoded_values_all_subsets]
            else:
                self.decoded_values_all_subsets = [AuditedList() for _ in range(n_subsets)]
        else:
            self.decoded_values_all_subsets = decoded_values_all_subsets or [[] for _ in range(n_subsets)]

        self.decoded_values = [] if n_subsets == 0 else self.decoded_values_all_subsets[0]

        self.idx_value = 0  # only needed for encoder

        self.nbits_offset = 0  # 201
        self.scale_offset = 0  # 202

        self.nbits_of_new_refval = 0  # 203
        self.new_refvals = {}  # 2 03 255 to conclude, not cancel

        self.nbits_of_associated = []  # 204
        self.nbits_of_skipped_local_descriptor = 0  # 206

        self.bsr_modifier = BSRModifier(
            nbits_increment=0, scale_increment=0, refval_factor=1
        )  # 207

        self.new_nbytes = 0  # 208

        self.data_not_present_count = 0  # 221
        self.status_qa_info_follows = QA_INFO_NA  # 222

        # bitmap definition
        self.bitmap = None
        self.bitmapped_descriptors = None
        self.bitmap_definition_state = BITMAP_NA
        self.most_recent_bitmap_is_for_reuse = False
        self.n_031031 = 0

        # Function to retrieve next bitmapped descriptor. Will be defined when
        # a bitmap is created or recalled.
        self.next_bitmapped_descriptor = None

        # Where to start count back for bitmap related descriptors
        self.back_reference_boundary = 0
        self.back_referenced_descriptors = None

    # noinspection PyAttributeOutsideInit
    def switch_subset_context(self, idx_subset):
        """
        This function is only useful for uncompressed data.
        """
        self.idx_subset = idx_subset
        # Reset new reference values to empty at start of each subset as anything defined
        # from previous subset should NOT affect this subset. Also we do not
        # care about what is defined in previous subset so we are not saving them.
        self.new_refvals = {}
        self.decoded_descriptors = self.decoded_descriptors_all_subsets[idx_subset]
        self.decoded_values = self.decoded_values_all_subsets[idx_subset]
        self.bitmap_links = self.bitmap_links_all_subsets[idx_subset]
        # Index to value is only needed for encoder
        self.idx_value = 0

    def mark_back_reference_boundary(self):
        self.back_reference_boundary = len(self.decoded_descriptors)

    def recall_bitmap(self):
        self.next_bitmapped_descriptor = functools.partial(next, iter(self.bitmapped_descriptors))
        return self.bitmap

    def cancel_bitmap(self):
        self.bitmap = None

    def cancel_all_back_references(self):
        self.back_referenced_descriptors = None
        self.bitmap = None
        self.bitmapped_descriptors = None

    def add_bitmap_link(self):
        """
        Must be called before the descriptor is processed
        """
        idx_descriptor, _ = self.next_bitmapped_descriptor()
        self.bitmap_links[len(self.decoded_descriptors)] = idx_descriptor

    def get_value_for_delayed_replication_factor(self, idx):
        if self.is_compressed:
            self._assert_equal_values_of_index(idx)
            value = self.decoded_values_all_subsets[0][idx]
        else:
            value = self.decoded_values[idx]

        if value is None or value < 0:
            raise PyBufrKitError('Delayed replication factor must be >= 0: got ({!r})'.format(value))

        return value

    def build_bitmapped_descriptors(self, bitmap):
        """
        Build the bitmapped descriptors based on the given bitmap. Also build
        the back referenced descriptors if it is not already defined.
        """
        # Second get all the back referenced descriptors if it does not already exist
        if not self.back_referenced_descriptors:
            self.back_referenced_descriptors = []
            for idx in range(self.back_reference_boundary - 1, -1, -1):
                descriptor = self.decoded_descriptors[idx]
                # The type has to be an exact match, not just isinstance
                if type(descriptor) is ElementDescriptor:
                    self.back_referenced_descriptors.insert(0, (idx, descriptor))
                    if len(self.back_referenced_descriptors) == len(bitmap):
                        break
        if len(self.back_referenced_descriptors) != len(bitmap):
            raise PyBufrKitError('Back referenced descriptors not matching defined Bitmap')

        # Lastly, get all the descriptors that has a corresponding Zero bit value
        self.bitmapped_descriptors = [
            (idx, d) for bit, (idx, d) in zip(
                bitmap,
                self.back_referenced_descriptors
            ) if bit == 0
        ]
        self.next_bitmapped_descriptor = functools.partial(next, iter(self.bitmapped_descriptors))

    def _assert_equal_values_of_index(self, idx):
        """
        Assert that the values of the specified index are identical for all
        subsets. It is only used for compressed data. For an example, to ensure
        the delayed replication factors are the same for all subsets.
        """
        minv, maxv = CoderState.minmax([values[idx] for values in self.decoded_values_all_subsets])
        assert minv == maxv, 'Values from all subsets are NOT identical'

    @staticmethod
    def minmax(values):
        """
        Give a list of values, find out the minimum and maximum, ignore any Nones.
        """
        mn, mx = None, None
        for v in values:
            if v is not None:
                if mn is None or mn > v:
                    mn = v
                if mx is None or mx < v:
                    mx = v
        return mn, mx
pybufrkit/templatecompiler.py
def process_compiled_template(coder, state, bit_operator, compiled_template):
    """
    This function runs the compiled code from the TemplateCompiler

    :param Coder coder:
    :param VmState state:
    :param bit_operator:
    :param Block compiled_template:
    """
    process_statements(coder, state, bit_operator, compiled_template.statements)
pybufrkit/constants.py
PARAMETER_TYPE_UNEXPANDED_DESCRIPTORS = 'unexpanded_descriptors'
pybufrkit/coder.py
class Coder(object):
    """
    This class is an abstract superclass for Decoder and Encoder. By itself it
    cannot do anything. But it provides common operations for subclasses.

    :param definitions_dir: Where to find the BPCL definition files.
    :param tables_root_dir: Where to find the BUFR table files.
    """

    def __init__(self,
                 definitions_dir=None,
                 tables_root_dir=None):

        self.section_configurer = SectionConfigurer(definitions_dir=definitions_dir)
        self.tables_root_dir = tables_root_dir or DEFAULT_TABLES_DIR

    @abc.abstractmethod
    def process(self, *args, **kwargs):
        """Entry point of the class"""

    @abc.abstractmethod
    def process_section(self, bufr_message, bit_operator, section):
        """
        Process the given section of a BUFR message

        :param bufr_message: The BufrMessage object to process
        :param bit_operator: The bit operator (reader or writer)
        :param section:
        """

    def process_template(self, state, bit_operator, template):
        """
        Process the top level BUFR Template

        :param state: The state of the processing.
        :param bit_operator: The bit operator for read/write bits.
        :param template: The BUFR Template of the message.
        """
        self.process_members(state, bit_operator, template.members)

    def process_members(self, state, bit_operator, members):
        """
        Process a list of descriptors that are members of a composite descriptor.

        :param state: The state of the processing.
        :param bit_operator: The bit operator for read/write bits.
        :param members: A list of descriptors.
        """
        for member in members:
            member_type = type(member)

            log.debug('Processing {} {}'.format(member, member.name if hasattr(member, 'name') else ''))

            # TODO: NOT using if-elif for following checks because they may co-exist???
            #      It is highly unlikely if not impossible

            # 221 YYY data not present for following YYY descriptors except class 0-9 and 31
            if state.data_not_present_count:
                state.data_not_present_count -= 1
                log.debug('Data not present: {} to go'.format(state.data_not_present_count))

                if member_type is ElementDescriptor:
                    X = member.X
                    if not (1 <= X <= 9 or X == 31):  # skipping
                        continue
                        # TODO: maybe the descriptor should still be kept and set its value to None?
                        #       So it helps to keep the structure intact??

            # Currently defining new reference values
            # For ElementDescriptor only. This makes sense though not explicitly stated in the manual
            if state.nbits_of_new_refval and member_type is ElementDescriptor:
                self.process_define_new_refval(state, bit_operator, member)
                continue

            # 206 YYY signify data width for local descriptor
            if state.nbits_of_skipped_local_descriptor:
                self.process_skipped_local_descriptor(state, bit_operator, member)
                continue

            # Currently defining new bitmap
            if state.bitmap_definition_state != BITMAP_NA:
                self.process_bitmap_definition(state, bit_operator, member)

            # Now process normally
            if member_type is ElementDescriptor:
                self.process_element_descriptor(state, bit_operator, member)

            elif member_type is FixedReplicationDescriptor:
                self.process_fixed_replication_descriptor(state, bit_operator, member)

            elif member_type is DelayedReplicationDescriptor:
                self.process_delayed_replication_descriptor(state, bit_operator, member)

            elif member_type is OperatorDescriptor:
                self.process_operator_descriptor(state, bit_operator, member)

            elif member_type is SequenceDescriptor:
                self.process_sequence_descriptor(state, bit_operator, member)

            else:
                raise UnknownDescriptor('Cannot process descriptor {} of type: {}'.format(
                    member, member_type.__name__))

    def process_define_new_refval(self, state, bit_operator, descriptor):
        """
        Process defining a new reference value for the given descriptor.

        :param state:
        :param bit_operator:
        :param descriptor:
        """
        log.debug('Defining new reference value for {}'.format(descriptor))
        if descriptor.unit == UNITS_STRING:
            raise PyBufrKitError('Cannot define new reference value for descriptor of string value')
        self.process_new_refval(state, bit_operator, descriptor, state.nbits_of_new_refval)

    def process_skipped_local_descriptor(self, state, bit_operator, descriptor):
        """
        Skip number of bits defined for the local descriptor.

        :param state:
        :param bit_operator:
        :param descriptor:
        """
        log.debug('Skipping {} bits for local descriptor {}'.format(
            state.nbits_of_skipped_local_descriptor, descriptor))

        # TODO: possible associated fields?
        self.process_codeflag(
            state, bit_operator,
            SkippedLocalDescriptor(descriptor.id, state.nbits_of_skipped_local_descriptor),
            state.nbits_of_skipped_local_descriptor
        )
        state.nbits_of_skipped_local_descriptor = 0  # reset back to zero

    def process_bitmap_definition(self, state, bit_operator, descriptor):
        """
        Process bitmap definition. This is basically done as a state machine
        for processing all the bits associated to the bitmap.

        :type state: CoderState
        :param bit_operator:
        :param descriptor:
        """
        if state.bitmap_definition_state == BITMAP_INDICATOR:
            # TODO: 236000 and 237000 are handled here. bad?
            if descriptor.id == 236000:  # bitmap defined for reuse
                log.debug('Defining bitmap for reuse')
                state.most_recent_bitmap_is_for_reuse = True
                state.bitmap_definition_state = BITMAP_WAITING_FOR_BIT
                state.n_031031 = 0

            elif descriptor.id == 237000:  # re-call most recent definition
                log.debug('Recall most recently defined bitmap')
                state.bitmap_definition_state = BITMAP_NA

            else:  # direct bitmap definition (non-reuse)
                log.debug('Defining non-reuse bitmap')
                state.most_recent_bitmap_is_for_reuse = False
                state.bitmap_definition_state = BITMAP_WAITING_FOR_BIT
                state.n_031031 = 0

        elif state.bitmap_definition_state == BITMAP_WAITING_FOR_BIT:
            if descriptor.id == 31031:
                state.bitmap_definition_state = BITMAP_BIT_COUNTING
                state.n_031031 += 1

        elif state.bitmap_definition_state == BITMAP_BIT_COUNTING:
            if descriptor.id == 31031:
                state.n_031031 += 1
            else:
                # TODO: for compressed data, ensure all bitmap is equal
                log.debug('Bitmap defined with {} bits'.format(state.n_031031))
                self.define_bitmap(state, state.most_recent_bitmap_is_for_reuse)
                state.bitmap_definition_state = BITMAP_NA

    def process_element_descriptor(self, state, bit_operator, descriptor):
        """
        Process an ElementDescriptor.

        :type state: CoderState
        :type bit_operator:
        :type descriptor: ElementDescriptor
        """
        X = descriptor.X

        # Read associated field if exists
        # Page 79 of layer 3 Guide, operators do not apply to class 31 element descriptor
        if state.nbits_of_associated and X != 31:
            log.debug('Processing associated field of {} bits'.format(state.nbits_of_associated))
            self.process_associated_field(state, bit_operator, descriptor)

        # Handle class 33 codes for QA information follows 222000 operator
        if X == 33:
            if state.status_qa_info_follows == QA_INFO_WAITING:
                state.status_qa_info_follows = QA_INFO_PROCESSING
            # Add the link between the QA info and its corresponding descriptor
            if state.status_qa_info_follows == QA_INFO_PROCESSING:
                state.add_bitmap_link()
        else:
            if state.status_qa_info_follows == QA_INFO_PROCESSING:
                state.status_qa_info_follows = QA_INFO_NA

        # Now we can process the element normally
        if descriptor.unit == UNITS_STRING:
            nbytes = state.new_nbytes if state.new_nbytes else descriptor.nbits // 8
            self.process_string(state, bit_operator, descriptor, nbytes)

        elif descriptor.unit in (UNITS_FLAG_TABLE, UNITS_CODE_TABLE):
            self.process_codeflag(state, bit_operator, descriptor, descriptor.nbits)

        else:
            nbits = (descriptor.nbits +
                     state.nbits_offset +
                     state.bsr_modifier.nbits_increment)
            scale = (descriptor.scale +
                     state.scale_offset +
                     state.bsr_modifier.scale_increment)
            scale_powered = 1.0 * 10 ** scale

            if descriptor.id not in state.new_refvals:  # no new refval is defined for this descriptor
                refval = descriptor.refval * state.bsr_modifier.refval_factor
                self.process_numeric(state, bit_operator, descriptor, nbits, scale_powered, refval)

            else:  # a new refval is defined for the descriptor, it must be retrieved at runtime
                self.process_numeric_of_new_refval(state, bit_operator,
                                                   descriptor, nbits, scale_powered,
                                                   state.bsr_modifier.refval_factor)

    def process_fixed_replication_descriptor(self, state, bit_operator, descriptor):
        """
        Process a fixed replication descriptor including all members belong to this
        replication structure.

        :param state:
        :param bit_operator:
        :type descriptor: FixedReplicationDescriptor
        """
        for _ in range(descriptor.n_repeats):
            self.process_members(state, bit_operator, descriptor.members)

    def process_delayed_replication_descriptor(self, state, bit_operator, descriptor):
        """
        Process the delayed replication factor descriptor.

        :param state:
        :param bit_operator:
        :type descriptor: DelayedReplicationDescriptor
        """
        # TODO: delayed repetition descriptor 031011, 031012
        if descriptor.id in (31011, 31012):
            raise NotImplementedError('delayed repetition descriptor')

        log.debug('Processing {}'.format(descriptor.factor))
        self.process_element_descriptor(state, bit_operator, descriptor.factor)
        for _ in range(self.get_value_for_delayed_replication_factor(state)):
            self.process_members(state, bit_operator, descriptor.members)

    def process_operator_descriptor(self, state, bit_operator, descriptor):
        """
        Process Operator Descriptor.

        :param state:
        :param bit_operator:
        :type descriptor: OperatorDescriptor
        """
        operator_code, operand_value = descriptor.operator_code, descriptor.operand_value

        if operator_code == 201:  # nbits offset
            state.nbits_offset = (operand_value - 128) if operand_value else 0

        elif operator_code == 202:  # scale offset
            state.scale_offset = (operand_value - 128) if operand_value else 0

        elif operator_code == 203:  # new reference value
            if operand_value == 255:  # 255 is to conclude not cancel
                state.nbits_of_new_refval = 0
            else:
                state.nbits_of_new_refval = operand_value
                if operand_value == 0:
                    state.new_refvals = {}

        elif operator_code == 204:  # associated field
            if operand_value == 0:
                state.nbits_of_associated.pop()
            else:
                state.nbits_of_associated.append(operand_value)

        elif operator_code == 205:  # read string of YYY bytes
            # TODO: Need take care of associated field?
            # TODO: this is not affected by nbytes_new 208 YYY
            self.process_string(state, bit_operator, descriptor, operand_value)

        elif operator_code == 206:  # skip local descriptor of YYY bits
            state.nbits_of_skipped_local_descriptor = operand_value

        elif operator_code == 207:  # increase nbits, scale, refval
            if operand_value == 0:
                state.bsr_modifier = BSRModifier(
                    nbits_increment=0, scale_increment=0, refval_factor=1
                )
            else:
                state.bsr_modifier = BSRModifier(
                    nbits_increment=(10 * operand_value + 2) // 3,
                    scale_increment=operand_value,
                    refval_factor=10 ** operand_value,
                )

        elif operator_code == 208:  # change all string type descriptor length
            state.new_nbytes = operand_value

        # Data not present for following YYY descriptors except class 0-9 and 31
        elif operator_code == 221:
            state.data_not_present_count = operand_value

        # Quality info, substituted, 1st order stats, difference stats, replaced
        elif operator_code in (222, 223, 224, 225, 232):
            if operand_value == 0:
                state.bitmap_definition_state = BITMAP_INDICATOR
                state.mark_back_reference_boundary()
                self.process_constant(state, bit_operator, descriptor, 0)
                if operator_code == 222:
                    state.status_qa_info_follows = QA_INFO_WAITING
            else:  # 255 for markers (this does not apply to 222)
                self.process_marker_operator_descriptor(state, bit_operator, descriptor)

        elif operator_code == 235:
            state.cancel_all_back_references()

        elif operator_code == 236:
            self.process_constant(state, bit_operator, descriptor, 0)

        elif operator_code == 237:
            if operand_value == 0:
                state.recall_bitmap()

            else:  # 255 cancel re-used bitmap
                if state.most_recent_bitmap_is_for_reuse:
                    state.cancel_bitmap()
            self.process_constant(state, bit_operator, descriptor, 0)

        else:  # TODO: 241, 242, 243
            raise NotImplementedError('Operator Descriptor {} not implemented'.format(descriptor))

    def process_sequence_descriptor(self, state, bit_operator, descriptor):
        self.process_members(state, bit_operator, descriptor.members)

    def process_associated_field(self, state, bit_operator, descriptor):
        """

        :type state: CoderState
        :param bit_operator:
        :param descriptor:
        """
        nbits_associated = sum(state.nbits_of_associated)
        self.process_codeflag(state, bit_operator,
                              AssociatedDescriptor(descriptor.id, nbits_associated),
                              nbits_associated)

    def process_marker_operator_descriptor(self, state, bit_operator, descriptor):
        """

        :type state: CoderState
        :param bit_operator:
        :param descriptor:
        """
        # TODO: do we really need associated field for marker operators
        if state.nbits_of_associated:
            self.process_associated_field(state, bit_operator, descriptor)
        self.process_bitmapped_descriptor(state, bit_operator, descriptor)

    def process_bitmapped_descriptor(self, state, bit_operator, descriptor):
        """
        A generic method for processing bitmapped descriptors. It is wrapped by
        providing different funcs to handle encoding and decoding for
        uncompressed and compressed data.
        """

        idx_descriptor, bitmapped_descriptor = state.next_bitmapped_descriptor()
        state.bitmap_links[len(state.decoded_descriptors)] = idx_descriptor

        # difference statistical values marker has different refval and nbits values
        if descriptor.id == 225255:
            bitmapped_descriptor = MarkerDescriptor.from_element_descriptor(
                bitmapped_descriptor,
                descriptor.id,
                refval=-2 ** bitmapped_descriptor.nbits,
                nbits=bitmapped_descriptor.nbits + 1,
            )
        else:
            bitmapped_descriptor = MarkerDescriptor.from_element_descriptor(
                bitmapped_descriptor,
                descriptor.id,
            )

        self.process_element_descriptor(state, bit_operator, bitmapped_descriptor)

    @abc.abstractmethod
    def get_value_for_delayed_replication_factor(self, state):
        """
        Get value of the latest delayed replication factor. This is called when
        processing through the Template. But the actual implementation will be
        provided by sub-classes.

        :param state:
        :return: The value for the latest processed delayed replication factor
        """

    @abc.abstractmethod
    def define_bitmap(self, state, reuse):
        """
        Define a bit map.

        :param state:
        :param reuse: Is this bitmap for reuse?
        """

    @abc.abstractmethod
    def process_numeric(self, state, bit_operator, descriptor, nbits, scale_powered, refval):
        """
        Process a descriptor that has numeric value.

        :param descriptor: A BUFR descriptor that has numeric value
        :param nbits: Number of bits to process for the descriptor.
        :param scale_powered: 10 to the scale factor power, i.e. 10 ** scale
        :param refval: The reference value
        """

    @abc.abstractmethod
    def process_string(self, state, bit_operator, descriptor, nbytes):
        """
        Process a descriptor that has string value

        :param descriptor: The BUFR descriptor
        :param nbytes: Number of BYTES to process for the descriptor.
        """

    @abc.abstractmethod
    def process_codeflag(self, state, bit_operator, descriptor, nbits):
        """
        Process a descriptor that has code/flag value. A code/flag value
        does not need to scale and refval.

        :param descriptor: The BUFR descriptor
        :param nbits: Number of bits to process for the descriptor.
        """

    @abc.abstractmethod
    def process_new_refval(self, state, bit_operator, descriptor, nbits):
        """
        Process the new reference value for the given descriptor.

        :param descriptor: The BUFR descriptor.
        :param nbits: Number of bits to process.
        """

    @abc.abstractmethod
    def process_numeric_of_new_refval(self, state, bit_operator,
                                      descriptor, nbits, scale_powered,
                                      refval_factor):
        """
        Process a descriptor that has numeric value with new reference value.

        :param descriptor: The BUFR descriptor.
        :param nbits: Number of bits to process for the descriptor.
        :param scale_powered: 10 to the scale factor power, i.e. 10 ** scale
        :param refval_factor: The factor to be applied to the new refval.
        """

    @abc.abstractmethod
    def process_constant(self, state, bit_operator, descriptor, value):
        """
        Process a constant, with no bit operations, for the given descriptor.

        :param descriptor: The BUFR descriptor.
        :param value: The constant value.
        """
pybufrkit/descriptors.py
class Descriptor(object):
    """
    This class is the base class of all BUFR descriptors. It provides common
    machinery for Descriptors.

    :param int id_: The descriptor ID.
    """

    def __init__(self, id_):
        self.id = id_

    def __str__(self):
        return '{:06d}'.format(self.id)

    def __repr__(self):
        return '<{}>'.format(self)

    def __eq__(self, other):
        return isinstance(other, self.__class__) and self.id == other.id

    @property
    def F(self):
        """
        The F value of the descriptor.
        """
        return self.id // 100000

    @property
    def X(self):
        """
        The X value of the descriptor.
        """
        return self.id // 1000 % 100

    @property
    def Y(self):
        """
        THe Y value of the descriptor.
        """
        return self.id % 1000

    def accept(self, visitor):
        visitor.visit_descriptor(self)
pybufrkit/constants.py
NUMERIC_MISSING_VALUES = [2 ** i - 1 for i in range(33)]
pybufrkit/templatecompiler.py
class CompiledTemplateManager(object):
    """
    A management class for compiled templates that handles caching and lookup.

    :param cache_max: The maximum number of compiled templates to cache.
    """

    def __init__(self, cache_max):
        self.template_compiler = TemplateCompiler()
        self.cache_max = cache_max
        self.cache = {}

    def get_or_compile(self, template, table_group):
        """

        :param descriptors.BufrTemplate template: The BUFR template to compile
        :param tables.TableGroup table_group: The Table Group used to instantiate the Template.
        :return:
        """
        key_of_compiled_template = (
            tuple(template.original_descriptor_ids),
            table_group.key
        )
        log.debug('Getting compiled template of key: {}'.format(key_of_compiled_template))
        compiled_template = self.cache.get(key_of_compiled_template, None)

        if compiled_template is None:
            log.debug('Cached version not available. Compiling now ...')
            compiled_template = self.template_compiler.process(template, table_group)

            if self.cache_max > 0:
                # TODO: Better cache invalidate algorithm
                if len(self.cache) >= self.cache_max:
                    self.cache.popitem()

                self.cache[key_of_compiled_template] = compiled_template

        return compiled_template
pybufrkit/constants.py
NBITS_PER_BYTE = 8
pybufrkit/bitops.py
def get_bit_writer():
    """
    Initialise and return a BitWriter.

    :return: BitWriter
    """
    return BitStringBitWriter()
pybufrkit/errors.py
class PyBufrKitError(Exception):
    """
    This is the root exception object of the package.
    """

    def __init__(self, message=''):
        self.message = message

    def __str__(self):
        return 'Error: {}'.format(self.message)
pybufrkit/bufr.py
class BufrMessage(object):
    """
    This class represents a single BUFR message that is comprised of different
    sections. Note this is different from BufrTemplateData which is only part of
    the overall message and dedicates to data associated to the Template.

    Properties of this class are proxies to actual fields of its sections. They
    are set by the sections when they are processed. The proxy approach allows
    these properties to be referenced in a consistent way no matter where they
    actually come from. This makes sections loosely coupled, i.e. one section
    does not need to know about other sections, and free to change if needed.
    """

    def __init__(self, filename=''):
        self.filename = filename
        self.sections = []
        self.serialized_bytes = None
        self.table_group_key = None
        self._edition = None
        # Default to zero for edition 1 which does not have this parameter
        self._master_table_number = SectionParameter('master_table_number',
                                                     0, 'unit', None, True, 0)
        # Default to zero for edition 1 and 2 which do not have this parameter
        self._originating_subcentre = SectionParameter('originating_subcentre',
                                                       0, 'unit', None, True, 0)

    def add_section(self, section):
        """
        Add a section to the message

        :param BufrSection section: The Bufr Section to add
        """
        self.sections.append(section)

    def build_template(self, tables_root_dir, normalize=1):
        """
        Build the BufrTemplate object using the list of unexpanded descriptors
        and corresponding table group.

        :param tables_root_dir: The root directory to find BUFR tables
        :param normalize: Whether to use some default table group if the specific
            one is not available.
        :return: A tuple of BufrTemplate and the associated TableGroup
        """
        table_group = TableGroupCacheManager.get_table_group(
            tables_root_dir=tables_root_dir,
            master_table_number=self.master_table_number.value,
            originating_centre=self.originating_centre.value,
            originating_subcentre=self.originating_subcentre.value,
            master_table_version=self.master_table_version.value,
            local_table_version=self.local_table_version.value,
            normalize=normalize
        )
        self.table_group_key = table_group.key

        return table_group.template_from_ids(*self.unexpanded_descriptors.value), table_group

    def wire(self):
        """
        Wire the flat list of descriptors and values to a full hierarchical
        structure. Also allocate all attributes to their corresponding
        descriptors.
        """
        self.template_data.value.wire()

    def subset(self, subset_indices):
        if max(subset_indices) >= self.n_subsets.value:
            raise PyBufrKitError('maximum subset index out of range')
        if min(subset_indices) < 0:
            raise PyBufrKitError('minimum subset index out of range')

        data = []
        for section in self.sections:
            section_data = []
            for parameter in section:
                if parameter.type == PARAMETER_TYPE_TEMPLATE_DATA:
                    section_data.append(
                        [v for i, v in enumerate(parameter.value.decoded_values_all_subsets)
                         if i in subset_indices]
                    )
                else:
                    section_data.append(
                        len(subset_indices) if parameter.name == 'n_subsets'
                        else parameter.value
                    )
            data.append(section_data)
        return data

    # Proxy properties follows
    @property
    def length(self):
        return self._length

    @length.setter
    def length(self, new_value):
        self._length = new_value

    @property
    def edition(self):
        return self._edition

    @edition.setter
    def edition(self, new_value):
        self._edition = new_value

    @property
    def master_table_number(self):
        return self._master_table_number

    @master_table_number.setter
    def master_table_number(self, new_value):
        self._master_table_number = new_value

    @property
    def originating_centre(self):
        return self._originating_centre

    @originating_centre.setter
    def originating_centre(self, new_value):
        self._originating_centre = new_value

    @property
    def originating_subcentre(self):
        return self._originating_subcentre

    @originating_subcentre.setter
    def originating_subcentre(self, new_value):
        self._originating_subcentre = new_value

    @property
    def master_table_version(self):
        return self._master_table_version

    @master_table_version.setter
    def master_table_version(self, new_value):
        self._master_table_version = new_value

    @property
    def local_table_version(self):
        return self._local_table_version

    @local_table_version.setter
    def local_table_version(self, new_value):
        self._local_table_version = new_value

    @property
    def timestamp(self):
        # TODO: 1900 correct?
        return datetime(
            self.year.value if self.edition.value >= 4 else self.year.value + 1900,
            self.month.value, self.day.value,
            self.hour.value, self.minute.value, self.second.value
        )

    @property
    def year(self):
        return self._year

    @year.setter
    def year(self, new_value):
        self._year = new_value

    @property
    def month(self):
        return self._month

    @month.setter
    def month(self, new_value):
        self._month = new_value

    @property
    def day(self):
        return self._day

    @day.setter
    def day(self, new_value):
        self._day = new_value

    @property
    def hour(self):
        return self._hour

    @hour.setter
    def hour(self, new_value):
        self._hour = new_value

    @property
    def minute(self):
        return self._minute

    @minute.setter
    def minute(self, new_value):
        self._minute = new_value

    @property
    def second(self):
        return self._second

    @second.setter
    def second(self, new_value):
        self._second = new_value

    @property
    def is_section2_presents(self):
        return self._is_section2_presents

    @is_section2_presents.setter
    def is_section2_presents(self, new_value):
        self._is_section2_presents = new_value

    @property
    def data_category(self):
        return self._data_category

    @data_category.setter
    def data_category(self, new_value):
        self._data_category = new_value

    @property
    def n_subsets(self):
        return self._n_subsets

    @n_subsets.setter
    def n_subsets(self, new_value):
        self._n_subsets = new_value

    @property
    def is_observation(self):
        return self._is_observation

    @is_observation.setter
    def is_observation(self, new_value):
        self._is_observation = new_value

    @property
    def is_compressed(self):
        return self._is_compressed

    @is_compressed.setter
    def is_compressed(self, new_value):
        self._is_compressed = new_value

    @property
    def unexpanded_descriptors(self):
        return self._unexpanded_descriptors

    @unexpanded_descriptors.setter
    def unexpanded_descriptors(self, new_value):
        self._unexpanded_descriptors = new_value

    @property
    def template_data(self):
        return self._template_data

    @template_data.setter
    def template_data(self, new_value):
        self._template_data = new_value


Question: import functools
import json
import logging
import six
from pybufrkit.constants import (BITPOS_START,
                                 NBITS_FOR_NBITS_DIFF,
                                 NBITS_PER_BYTE,
                                 NUMERIC_MISSING_VALUES,
                                 PARAMETER_TYPE_TEMPLATE_DATA,
                                 PARAMETER_TYPE_UNEXPANDED_DESCRIPTORS)
from pybufrkit.errors import PyBufrKitError
from pybufrkit.bitops import get_bit_writer
from pybufrkit.bufr import BufrMessage
from pybufrkit.templatedata import TemplateData
from pybufrkit.descriptors import Descriptor
from pybufrkit.coder import CoderState, Coder
from pybufrkit.templatecompiler import CompiledTemplateManager, process_compiled_template
"""
pybufrkit.encoder
~~~~~~~~~~~~~~~~~

"""
from __future__ import absolute_import
from __future__ import print_function



__all__ = ['Encoder']

log = logging.getLogger(__file__)


def nbits_for_uint(x):
    """
    Given the maximum difference, find out the number of bits needed for this number
    and also considering the missing value (all ones).
    """
    binx = bin(x)[2:]
    nbits = len(binx)
    # If this number has all bits of one, the required length is one bit longer
    # as all 1s is for missing value.
    if binx.count('1') == len(binx):
        nbits += 1

    return nbits


class Encoder(Coder):
    """
    The encoder takes a JSON object or string and encoded it to a BUFR message.

    :param ignore_declared_length: If set, ignore the section_length declared
        in the input JSON message and always calculated it.
    """

    def __init__(self,
                 definitions_dir=None,
                 tables_root_dir=None,
                 ignore_declared_length=True,
                 compiled_template_cache_max=None,
                 master_table_number=None,
                 master_table_version=None):

        super(Encoder, self).__init__(definitions_dir, tables_root_dir)
        self.ignore_declared_length = ignore_declared_length
        self.overrides = {}
        if master_table_number:
            self.overrides['master_table_number'] = master_table_number
        if master_table_version:
            self.overrides['master_table_version'] = master_table_version

        # Only enable template compilation if cache is requested
        if compiled_template_cache_max is not None:
            self.compiled_template_manager = CompiledTemplateManager(compiled_template_cache_max)
            log.debug('Template compilation enabled with cache size of {}'.format(compiled_template_cache_max))
        else:
            self.compiled_template_manager = None

    def process(self, s, file_path='<string>', wire_template_data=True):
        """
        Entry point for the encoding process. The process encodes a JSON format
        message to BUFR message.

        :param s: A JSON or its string serialized form
        :param file_path: The file path to the JSON file.
        :param wire_template_data: Whether to wire the template data to construct
            a fully hierarchical structure from the flat lists.

        :return: A bitstring object of the encoded message.
        """

        if isinstance(s, (six.binary_type, six.text_type)):
            # TODO: ensure all strings are loaded as plain ascii instead of unicode from JSON
            json_data = json.loads(s) if six.PY3 else json.loads(s, encoding='latin-1')
        else:
            json_data = s

        bit_writer = get_bit_writer()
        bufr_message = BufrMessage(filename=file_path)

        nbits_encoded = 0
        section_index = 0
        # When a section is not present in the json data. The index must not be
        # increase for the section.
        index_offset = 0
        while True:
            section = self.section_configurer.configure_section_with_values(
                bufr_message, section_index, json_data[section_index - index_offset],
                self.overrides)
            section_index += 1
            if section is None:  # optional section is not present
                index_offset += 1  # This section should not be counted
                continue
            nbits_encoded += self.process_section(bufr_message, bit_writer, section)
            if section.end_of_message:
                break

        # A zero length means the actual length must be calculated
        # Fix is needed for both the message object and the serialized bits
        nbytes_write = bit_writer.get_pos() // NBITS_PER_BYTE
        if bufr_message.length.value == 0 or self.ignore_declared_length:
            bufr_message.length.value = nbytes_write
            section = bufr_message.length.parent
            bit_writer.set_uint(
                bufr_message.length.value,
                bufr_message.length.nbits,

Answer:                 section.get_metadata(BITPOS_START) + section.get_parameter_offset('length')