nn_dataflow/tests/pipeline_test/test_pipeline_fixture.py
class TestPipelineFixture(unittest.TestCase):
    ''' Base fixture class for layer pipeline tests. '''

    def setUp(self):

        self.net = {}

        net = Network('net1')
        # Linear.
        net.set_input_layer(InputLayer(10, 1))
        net.add('0', FCLayer(10, 20))
        net.add('1', FCLayer(20, 30))
        net.add('1p', PoolingLayer(30, 1, 1))
        net.add('2', FCLayer(30, 40))
        net.add('3', FCLayer(40, 50))
        self.net[net.net_name] = net

        net = Network('net2')
        # Long linear.
        net.set_input_layer(InputLayer(1, 1))
        for idx in range(16):
            net.add(str(idx), FCLayer(1, 1))
        self.net[net.net_name] = net

        net = Network('net3')
        # Fork.
        # /0-2\   /6- 7- 8\
        #   x  4-5         12
        # \1-3/   \9-10-11/
        net.set_input_layer(InputLayer(1, 1))
        net.add('0', FCLayer(1, 1), prevs=net.INPUT_LAYER_KEY)
        net.add('1', FCLayer(1, 1), prevs=net.INPUT_LAYER_KEY)
        net.add('2', FCLayer(2, 1), prevs=('0', '1'))
        net.add('2p', PoolingLayer(1, 1, 1))
        net.add('3', FCLayer(2, 1), prevs=('0', '1'))
        net.add('4', FCLayer(2, 1), prevs=('2p', '3'))
        net.add('5', FCLayer(1, 1))
        net.add('5p', PoolingLayer(1, 1, 1))
        net.add('6', FCLayer(1, 1), prevs='5p')
        net.add('7', FCLayer(1, 1))
        net.add('8', FCLayer(1, 1))
        net.add('9', FCLayer(1, 1), prevs='5p')
        net.add('10', FCLayer(1, 1))
        net.add('11', FCLayer(1, 1))
        net.add('12', FCLayer(2, 1), prevs=('8', '11'))
        self.net[net.net_name] = net

        net = Network('net4')
        # Complex fork.
        #          /5       \
        # 0-1-2-3-4-6-7-8-10-14
        #              \9/
        #          \11-12   /
        #          \13      /
        net.set_input_layer(InputLayer(1, 1))
        net.add('0', FCLayer(1, 1))
        net.add('1', FCLayer(1, 1))
        net.add('2', FCLayer(1, 1))
        net.add('3', FCLayer(1, 1))
        net.add('4', FCLayer(1, 1))
        net.add('5', FCLayer(1, 1), prevs='4')
        net.add('6', FCLayer(1, 1), prevs='4')
        net.add('7', FCLayer(1, 1))
        net.add('8', FCLayer(1, 1), prevs='7')
        net.add('9', FCLayer(1, 1), prevs='7')
        net.add('10', FCLayer(1, 1))
        net.add('10p', PoolingLayer(2, 1, 1), prevs=('8', '10'))
        net.add('11', PoolingLayer(1, 1, 1), prevs='4')
        net.add('12', FCLayer(1, 1))
        net.add('13', PoolingLayer(1, 1, 1), prevs='4')
        net.add('14', FCLayer(5, 1), prevs=('5', '10p', '12', '13'))
        self.net[net.net_name] = net

        net = Network('net5')
        # Corner cases.
        #  ----\
        # //1-2\ 7-8\
        # 0-3-4-x   10-11-12
        #  \ \5/ 9 /  \__/
        #   6--/
        net.set_input_layer(InputLayer(1, 1))
        net.add('0', FCLayer(1, 1))
        net.add('1', FCLayer(1, 1), prevs='0')
        net.add('2', FCLayer(1, 1))
        net.add('3', FCLayer(1, 1), prevs='0')
        net.add('4', FCLayer(1, 1), prevs='3')
        net.add('5', FCLayer(1, 1), prevs='3')
        net.add('6', FCLayer(1, 1), prevs='0')
        net.add('7', FCLayer(5, 1), prevs=('0', '2', '4', '5', '6'))
        net.add('8', FCLayer(1, 1))
        net.add('9', FCLayer(5, 1), prevs=('0', '2', '4', '5', '6'))
        net.add('10', FCLayer(2, 1), prevs=('8', '9'))
        net.add('11', FCLayer(1, 1))
        net.add('12', FCLayer(2, 1), prevs=('10', '11'))
        self.net[net.net_name] = net

        net = Network('net6')
        # Fmap sizes.
        net.set_input_layer(InputLayer(1, 24))
        net.add('0', ConvLayer(1, 1, 24, 3))
        net.add('1', ConvLayer(1, 1, 12, 3, strd=2))
        net.add('1p', PoolingLayer(1, 6, 2))
        net.add('2', ConvLayer(1, 1, 6, 3))
        net.add('3', ConvLayer(1, 1, 6, 3, strd=4), prevs=('0'))
        self.net[net.net_name] = net

        net = Network('net7')
        # Topological order: see a visited vertex again.
        #  /---
        # 0-1-\\
        #  \2--2p
        net.set_input_layer(InputLayer(1, 1))
        net.add('0', FCLayer(1, 1))
        net.add('1', FCLayer(1, 1), prevs='0')
        net.add('2', FCLayer(1, 1), prevs='0')
        net.add('2p', PoolingLayer(3, 1, 1), prevs=('0', '1', '2'))
        self.net[net.net_name] = net

        net = Network('net8')
        # Forward to the middle.
        #    /-\
        # 0-1-2-2p-4-4p
        #  \-3------/
        net.set_input_layer(InputLayer(1, 1))
        net.add('0', FCLayer(1, 1))
        net.add('1', FCLayer(1, 1), prevs='0')
        net.add('2', FCLayer(1, 1), prevs='1')
        net.add('2p', PoolingLayer(2, 1, 1), prevs=('1', '2'))
        net.add('3', FCLayer(1, 1), prevs='0')
        net.add('4', FCLayer(2, 1), prevs='2p')
        net.add('4p', PoolingLayer(2, 1, 1), prevs=('3', '4'))
        self.net[net.net_name] = net

        net = Network('net9')
        # Previous layers include input and others.
        net.set_input_layer(InputLayer(1, 1))
        net.add('0', FCLayer(1, 1))
        net.add('1', FCLayer(2, 1), prevs=(net.INPUT_LAYER_KEY, '0'))
        self.net[net.net_name] = net

        # Real networks.
        for net_name in all_networks():
            self.net[net_name] = import_network(net_name)

        self.batch_size = 16

        self.resource = Resource(
            proc_region=NodeRegion(origin=PhyDim2(0, 0), dim=PhyDim2(8, 8),
                                   type=NodeRegion.PROC),
            dram_region=NodeRegion(origin=PhyDim2(0, 0), dim=PhyDim2(8, 8),
                                   type=NodeRegion.DRAM),
            src_data_region=NodeRegion(origin=PhyDim2(0, 0), dim=PhyDim2(8, 4),
                                       type=NodeRegion.DRAM),
            dst_data_region=NodeRegion(origin=PhyDim2(0, 4), dim=PhyDim2(8, 4),
                                       type=NodeRegion.DRAM),
            dim_array=PhyDim2(16, 16), size_gbuf=65536, size_regf=64,
            array_bus_width=float('inf'), dram_bandwidth=float('inf'),
            no_time_mux=False)

        part = PartitionScheme(order=range(pe.NUM), pdims=[(1, 1)] * pe.NUM)
        self.ofmap_layout = DataLayout(
            frngs=(FmapRange((0, 0, 0, 0), (2, 4, 16, 16)),),
            regions=(NodeRegion(origin=PhyDim2(0, 0), dim=PhyDim2(1, 1),
                                type=NodeRegion.DRAM),),
            parts=(part,))


    def _make_ilp(self, network):
        ''' Make an InterLayerPipeline instance. '''
        return InterLayerPipeline(network, self.batch_size, self.resource)

    def _make_segment(self, vseg, network, temporal=False, max_util_drop=None,
                      with_opt=True):
        ''' Convert vertex segment to (layer) segment. '''
        kwargs = {}
        if max_util_drop is not None:
            kwargs['max_util_drop'] = max_util_drop
        if not with_opt:
            kwargs['with_opt'] = False
        ilp = self._make_ilp(network)
        seg = tuple(ilp.dag_vertex_list[vidx] for vidx in vseg)
        if temporal:
            seg = (sum(seg, tuple()),)
        return PipelineSegment(seg, ilp.network, ilp.batch_size, ilp.resource,
                               **kwargs)

    def _make_sched_res(self, sched_seq, time, top_ti=1, top_to=1, top_tb=1,
                        top_ord=range(le.NUM), dram_time=0, num_nodes=4):
        scheme = OrderedDict()
        scheme['cost'] = 1.234 + 9.876
        scheme['time'] = max(time, dram_time)
        scheme['num_nodes'] = num_nodes
        scheme['proc_time'] = time
        scheme['bus_time'] = 0
        scheme['dram_time'] = dram_time
        scheme['ti'] = [top_ti, 1, 1]
        scheme['to'] = [top_to, 1, 1]
        scheme['tb'] = [top_tb, 1, 1]
        scheme['tvals'] = [[top_ti, top_to, top_tb], [1] * 3, [1] * 3]
        scheme['orders'] = [top_ord, range(le.NUM), range(le.NUM)]
        return SchedulingResult(scheme=scheme,
                                ofmap_layout=self.ofmap_layout,
                                sched_seq=sched_seq)

    def _gen_all_segment(self, network, **kwargs):
        '''
        Generate all segments directly from all layers and all vertex segments.
        '''
        # pylint: disable=protected-access
        ilp = self._make_ilp(network)
        for layer in network:
            yield PipelineSegment(((layer,),), ilp.network, ilp.batch_size,
                                  ilp.resource)
        for vseg in ilp._gen_vseg():
            segment = self._make_segment(vseg, network, **kwargs)
            if len(segment) == 1 and len(segment[0]) == 1:
                continue
            yield segment

    def _validate_allocation(self, segment, allocation):
        ''' Validate segment resource allocation. '''

        # Match segment.
        self.assertEqual(len(allocation), len(segment))
        for ltpl, rtpl in zip(segment, allocation):
            self.assertEqual(len(rtpl), len(ltpl))
            self.assertTrue(all(isinstance(r, Resource) for r in rtpl))

        # Number of nodes.
        nodes = []  # number of nodes.
        for rtpl in allocation:
            nodes.append(rtpl[0].proc_region.dim.size())
        self.assertEqual(sum(nodes), self.resource.proc_region.dim.size())

        # Temporal schedules share processing region; spatial schedules use
        # non-overlapped processing regions.
        used_proc_nodes = set()  # used processing nodes
        for rtpl in allocation:
            proc_region = rtpl[0].proc_region
            self.assertTrue(all(r.proc_region == proc_region for r in rtpl))
            for n in proc_region.iter_node():
                self.assertTrue(self.resource.proc_region.contains_node(n),
                                '_validate_allocation: node {} outside of '
                                'the processing region {}'
                                .format(n, self.resource.proc_region))
                self.assertNotIn(n, used_proc_nodes,
                                 '_validate_allocation: node {} has been '
                                 'used.'.format(n))
                used_proc_nodes.add(n)

        # Data liveness.
        data_regions = {}  # layers that have data currently on-chip
        for ltpl, rtpl in zip(segment, allocation):

            for l, r in zip(ltpl, rtpl):

                # Check data source.
                prev_layers = segment.network.prevs(l)

                for pl in prev_layers:
                    if pl not in data_regions:
                        # Previous layer is not on-chip, from memory.
                        # Try find a layer responsible to fetch shared mem src.
                        try:
                            sh_sp_idx = next((i for i in range(len(allocation))
                                              if allocation[i][0].proc_region
                                              == r.src_data_region))
                        except StopIteration:
                            # No shared mem src.
                            self.assertEqual(
                                r.src_data_region,
                                self.resource.src_data_region,
                                '_validate_allocation: layer {}\'s prev {} '
                                'is not on-chip, should be from {}, but {}.'
                                .format(l, pl, self.resource.src_data_region,
                                        r.src_data_region))
                        else:
                            # There exists shared mem src.
                            sh_l = segment[sh_sp_idx][0]
                            self.assertEqual(segment.network.prevs(l),
                                             segment.network.prevs(sh_l),
                                             '_validate_allocation: layer {} '
                                             'expects on-chip mem src sharing '
                                             'with {}, but prevs differ.'
                                             .format(l, sh_l))
                    elif data_regions[pl] != r.proc_region:
                        # Previous layer is on-chip and not local.
                        self.assertEqual(
                            r.src_data_region, data_regions[pl],
                            '_validate_allocation: layer {}\'s prev {} '
                            'is on-chip, should be from {}, but {}.'
                            .format(l, pl, data_regions[pl],
                                    r.src_data_region))

                # Update data based on destination.
                # Local or store back to memory. Both will be available on-chip.
                self.assertTrue(r.dst_data_region == r.proc_region
                                or r.dst_data_region
                                == self.resource.dst_data_region,
                                '_validate_allocation: data can only '
                                'be local or storing back to mem.')
                # Overwrite.
                local_node_set = set(r.proc_region.iter_node())
                data_regions = {pl: data_regions[pl] for pl in data_regions
                                if local_node_set.isdisjoint(
                                    data_regions[pl].iter_node())}
                data_regions[l] = r.proc_region

    def _validate_constraint(self, segment, constraint):
        ''' Validate segment scheduling constraint. '''
        # pylint: disable=too-many-branches

        # Match segment.
        self.assertEqual(len(constraint), len(segment))
        for ltpl, ctpl in zip(segment, constraint):
            self.assertEqual(len(ctpl), len(ltpl))
            self.assertTrue(all(isinstance(c, SchedulingConstraint)
                                for c in ctpl))

        # Same top tb.
        top_tb = constraint[0][0].topbat
        self.assertTrue(all(c.topbat == top_tb
                            for ctpl in constraint for c in ctpl))

        # Top tb is a factor of batch size.
        if top_tb:
            self.assertEqual((segment.batch_size) % top_tb, 0)

        # Data availability.

        seg_layers = set(l for ltpl in segment for l in ltpl)

        class OutAccPat():
            ''' Output data access pattern types. '''
            # pylint: disable=too-few-public-methods
            ANY = 0   # can access in any way
            DBF = -1  # must double-buffer
            # SEQ: use any positive value to represent sequential access with
            # certain number of groups.

        # Available data in each spatial subregions. Each is represented by a
        # tuple of layer name and its output data access pattern.
        avail_data = [(None, OutAccPat.ANY) for _ in segment]

        # Get groups of layers sharing the same memory source.
        prevs2layers = {}
        for ltpl in segment:
            l = ltpl[0]
            prevs2layers.setdefault(segment.network.prevs(l), []).append(l)
        sh_mem_src_groups = [ls for ps, ls in prevs2layers.items()
                             if not seg_layers.intersection(ps) and len(ls) > 1]
        sh_mem_src_topifms = [None] * len(sh_mem_src_groups)

        # Whether to defer fully buffering output.
        fb_out = False
        fb_out_conv = None

        for sp_idx, (ltpl, ctpl) in enumerate(zip(segment, constraint)):

            self.assertFalse(fb_out,
                             '_validate_constraint: deferring fully buffering '
                             'from {} should not cross spatial scheduling {}.'
                             .format(fb_out_conv, sp_idx - 1))

            for tm_idx, (layer, cstr) in enumerate(zip(ltpl, ctpl)):

                # Source data and their access patterns.
                prev_layers = segment.network.prevs(layer)
                prev_oaps = []
                for pl in prev_layers:
                    if pl not in seg_layers:
                        # Off-chip sources.
                        poap = OutAccPat.ANY
                    elif pl in ltpl:
                        # On-chip and local.
                        self.assertEqual(avail_data[sp_idx][0], pl,
                                         '_validate_constraint: layer {} ({}) '
                                         'local source data {} not available, '
                                         'maybe not the immediate previous.'
                                         .format(layer, (sp_idx, tm_idx), pl))
                        poap = avail_data[sp_idx][1]
                    else:
                        # On-chip and neighbor.
                        poap = next((avail_data[p_sp_idx][1]
                                     for p_sp_idx in range(sp_idx)
                                     if avail_data[p_sp_idx][0] == pl),
                                    None)
                        self.assertFalse(poap is None,
                                         '_validate_constraint: layer {} ({}) '
                                         'neighbor source data {} not '
                                         'available on-chip.'
                                         .format(layer, (sp_idx, tm_idx), pl))
                    prev_oaps.append(poap)
                # Only buffer input if having source on-chip.
                has_src = not seg_layers.isdisjoint(prev_layers)

                # The single SEQ source.
                seq = None
                # str is greater than all numbers, see
                # https://docs.python.org/2/library/stdtypes.html#comparisons
                seq_prev_oaps = [poap for poap in prev_oaps \
                                 if isinstance(poap, str) or \
                                    (isinstance(poap, int) and poap > 0)]
                if seq_prev_oaps:
                    self.assertEqual(len(seq_prev_oaps), 1,
                                     '_validate_constraint: layer {} ({}) '
                                     'has multiple SEQ input.'
                                     '\nsrcs: {}, oaps: {}'
                                     .format(layer, (sp_idx, tm_idx),
                                             prev_layers, prev_oaps))
                    seq = seq_prev_oaps[0]

                # Destination data.
                # Only buffer output if having destination on-chip.
                next_layers = segment.network.nexts(layer)
                has_dst = not seg_layers.isdisjoint(next_layers)

                # Validation.

                for g_idx, group in enumerate(sh_mem_src_groups):
                    if layer in group:
                        if sh_mem_src_topifms[g_idx] is None:
                            sh_mem_src_topifms[g_idx] = cstr.topifm
                        self.assertEqual(sh_mem_src_topifms[g_idx], cstr.topifm,
                                         '_validate_constraint: layer {} ({}) '
                                         'share memory source with {}, but has '
                                         'mismatched topifm {} with {}.'
                                         .format(layer, (sp_idx, tm_idx),
                                                 group, cstr.topifm,
                                                 sh_mem_src_topifms[g_idx]))
                        break
                else:
                    if not has_src:
                        self.assertEqual(cstr.topifm, 0,
                                         '_validate_constraint: layer {} ({}) '
                                         'should not constrain input as it '
                                         'does not have on-chip sources.'
                                         .format(layer, (sp_idx, tm_idx)))

                if isinstance(segment.network[layer], ConvLayer):

                    self.assertFalse(fb_out,
                                     '_validate_constraint: deferring fully '
                                     'buffering from {} has not been realized.'
                                     .format(fb_out_conv))

                    if any(pl in ltpl for pl in prev_layers):
                        # Local source.
                        lcl_poap = avail_data[sp_idx][1]
                        self.assertIn(lcl_poap, (OutAccPat.DBF, OutAccPat.ANY),
                                      '_validate_constraint: layer {} ({}) '
                                      'local source data {} must fully '
                                      'buffer output.'
                                      .format(layer, (sp_idx, tm_idx),
                                              lcl_poap))

                    # DBF source.
                    if OutAccPat.DBF in prev_oaps:
                        # Must fully buffer CONV input.
                        self.assertEqual(cstr.topifm, 1,
                                         '_validate_constraint: layer {} ({}) '
                                         'input is not fully buffered but has '
                                         'DBF source.\nsrcs: {}, oaps: {}'
                                         '\n{}'
                                         .format(layer, (sp_idx, tm_idx),
                                                 prev_layers, prev_oaps,
                                                 cstr))

                    # SEQ source.
                    if seq and has_dst:
                        # Cannot be lazily updated.
                        self.assertNotIsInstance(
                            seq, str,
                            '_validate_constraint: CONV layer {} ({}) cannot '
                            'use lazy update (from {})'
                            .format(layer, (sp_idx, tm_idx), seq))
                        # Must match SEQ.
                        self.assertEqual(cstr.topifm, seq,
                                         '_validate_constraint: layer {} ({}) '
                                         'input groups ({}) and its SEQ src '
                                         'output groups ({}) are mismatched.'
                                         '\nsrcs: {}, oaps: {}'
                                         .format(layer, (sp_idx, tm_idx),
                                                 cstr.topifm, seq,
                                                 prev_layers, prev_oaps))
                        # Also must fully buffer CONV output.
                        self.assertEqual(cstr.topofm, 1,
                                         '_validate_constraint: layer {} ({}) '
                                         'output is not fully buffered but has '
                                         'SEQ source.\nsrcs: {}, oaps: {}'
                                         .format(layer, (sp_idx, tm_idx),
                                                 prev_layers, prev_oaps))
                        # Deferred apply to the last layer in the group.
                        fb_out = True
                        fb_out_conv = layer

                    oap = None
                    if cstr.topofm == 1:
                        if cstr.topifm == 1:
                            # Fully buffer both, can access output in any way.
                            # This is fine as we require to buffer either input
                            # or output for CONV (see below).
                            oap = OutAccPat.ANY
                        else:
                            oap = OutAccPat.DBF
                    elif has_dst and cstr.topofm > 0:
                        oap = cstr.topofm
                        if has_src:
                            self.assertEqual(cstr.topifm, 1,
                                             '_validate_constraint: layer {} '
                                             '({}) has on-chip src and dst '
                                             'but neither input nor output '
                                             'are fully buffered.\ncstr: {}.'
                                             .format(layer, (sp_idx, tm_idx),
                                                     cstr))
                    elif has_dst:
                        # Lazy update, record layer name as seq.
                        oap = layer

                else:

                    # SEQ source.
                    if seq and has_dst:
                        # Must match SEQ, or fully buffer output.
                        self.assertTrue(cstr.topofm == seq or cstr.topofm == 1
                                        or seq in cstr.update_dict,
                                        '_validate_constraint: layer {} ({}) '
                                        'output is not fully buffered, and '
                                        'groups ({}) and its SEQ src output '
                                        'groups ({}) are mismatched, and '
                                        'lazy update is not used.'
                                        '\nsrcs: {}, oaps: {}'
                                        .format(layer, (sp_idx, tm_idx),
                                                cstr.topofm, seq,
                                                prev_layers, prev_oaps))

                    if cstr.topofm == 1:
                        # Fully buffer output.
                        oap = OutAccPat.DBF
                    elif isinstance(seq, str):
                        # Lazy update.
                        oap = seq
                    else:
                        # SEQ output.
                        oap = cstr.topofm

                # Realize deferred fully buffering output.
                if cstr.topofm == 1:
                    fb_out = False  # reset

                # Overwrite the previous temporal scheduling.
                avail_data[sp_idx] = (layer, oap)
nn_dataflow/core/network.py
class Network():
    '''
    NN topology. Support DAG structure of layers.
    '''

    INPUT_LAYER_KEY = '__INPUT__'

    def __init__(self, net_name):
        self.net_name = net_name
        self.layer_dict = OrderedDict()
        self.prevs_dict = {}
        self.nexts_dict = {}
        self.ext_dict = OrderedDict()

    def set_input_layer(self, input_layer):
        '''
        Set the input layer.
        '''
        if self.INPUT_LAYER_KEY in self.layer_dict:
            raise KeyError('Network: only one input layer is allowed.')

        if not isinstance(input_layer, InputLayer):
            raise TypeError('Network: input_layer must be an InputLayer '
                            'instance.')

        self.layer_dict[self.INPUT_LAYER_KEY] = input_layer

    def input_layer(self):
        '''
        Get the input layer.
        '''
        return self.layer_dict[self.INPUT_LAYER_KEY]

    def add(self, layer_name, layer, prevs=None):
        '''
        Add a named layer, with optional previous layer(s).

        If previous layer(s) is not given, assume it follows the last added
        layer.
        '''
        if self.INPUT_LAYER_KEY not in self.layer_dict:
            raise RuntimeError('Network: must first set input layer.')

        if layer_name in self.layer_dict:
            raise KeyError('Network: layer {} already exists.'
                           .format(layer_name))

        if not isinstance(layer, Layer):
            raise TypeError('Network: layer must be a Layer instance.')

        # First figure out previous layers.
        if prevs:
            # Ensure `prevs` as a tuple.
            if isinstance(prevs, str):
                prevs = (prevs,)
            else:
                prevs = tuple(prevs)
            # Ensure previous layers are already added.
            for p in prevs:
                try:
                    self.__getitem__(p)
                except KeyError:
                    raise KeyError('Network: given previous layer {} '
                                   'has not been added to the network'.
                                   format(p))
        else:
            prevs = (list(self.layer_dict.keys())[-1],)

        self.layer_dict[layer_name] = layer
        self.prevs_dict[layer_name] = prevs

        # Ensure dimension matching between layers.
        try:
            self._check_prevs(layer_name)
        except ValueError:
            del self.layer_dict[layer_name]
            del self.prevs_dict[layer_name]
            raise

        for p in prevs:
            self.nexts_dict.setdefault(p, []).append(layer_name)

    def add_ext(self, layer_name, layer):
        '''
        Add a named external layer.
        '''
        if layer_name in self.ext_dict:
            raise KeyError('Network: external layer {} already exists.'
                           .format(layer_name))

        if not isinstance(layer, InputLayer):
            raise TypeError('Network: external layer must be an InputLayer '
                            'instance.')

        self.ext_dict[layer_name] = layer

    def prevs(self, layer_name):
        '''
        Get the previous layers of the given layer name.

        Return a tuple of all the previous layer names. Use `None` to represent
        the input layer in the returned tuple.
        '''
        if layer_name == self.INPUT_LAYER_KEY:
            raise ValueError('Network: cannot get previous layers for '
                             'input layer.')
        if layer_name in self.ext_dict:
            raise ValueError('Network: cannot get previous layers for '
                             'external layers.')

        prevs = tuple(None if p == self.INPUT_LAYER_KEY else p
                      for p in self.prevs_dict[layer_name])
        assert prevs

        return prevs

    def nexts(self, layer_name):
        '''
        Get the next layers of the given layer name, i.e., the layers that need
        the output of this layer.

        Return a tuple of all the next layer names. Use `None` to represent the
        output of the last layer in the returned tuple.
        '''
        try:
            nexts = tuple(self.nexts_dict[layer_name])
        except KeyError:
            nexts = tuple([None])
        assert nexts

        return nexts

    def firsts(self):
        '''
        Get a tuple of the first layers, i.e., those with only the input layer
        or external layers as their previous layers.

        If a layer has other layers besides the input/external layers as its
        previous layers, it does not count as a first layer.
        '''
        input_ext_layers = set([None]).union(self.ext_layers())
        firsts = []
        for layer_name in self:
            prevs = self.prevs(layer_name)
            if input_ext_layers.issuperset(prevs):
                firsts.append(layer_name)
        return tuple(firsts)

    def lasts(self):
        '''
        Get a tuple of the last layers, i.e., those with no next layer.
        '''
        lasts = []
        for layer_name in self:
            nexts = self.nexts(layer_name)
            if nexts == (None,):
                lasts.append(layer_name)
        return tuple(lasts)

    def ext_layers(self):
        '''
        Get a tuple of the external layers.
        '''
        return tuple(self.ext_dict.keys())

    def _check_prevs(self, layer_name):
        '''
        Check the previous layers of the given layer name.
        '''
        layer = self.layer_dict[layer_name]

        prevs = self.prevs_dict[layer_name]
        assert prevs

        # Compare the ifmap dimensions of this layer, with all the ofmaps of
        # the previous layers.
        sum_nfmaps = 0

        for p in prevs:
            pl = self.__getitem__(p)

            # Ensure fmap sizes match. Allow padding.
            if not layer.is_valid_padding_sifm((pl.hofm, pl.wofm)):
                raise ValueError('Network: {}, a previous layer of {}, '
                                 'has mismatch fmap size: {} vs. {}.'
                                 .format(p, layer_name,
                                         (pl.hofm, pl.wofm),
                                         (layer.hofm, layer.wofm)))

            sum_nfmaps += pl.nofm

        if sum_nfmaps != layer.nifm:
            raise ValueError('Network: {} cannot be the previous layers of {}.'
                             .format(' | '.join(prevs), layer_name))

    def __contains__(self, layer_name):
        ''' Whether the network contains a layer. '''
        return layer_name in self.layer_dict or layer_name in self.ext_dict

    def __len__(self):
        ''' Number of layers in the network. '''
        if self.INPUT_LAYER_KEY not in self.layer_dict:
            assert not self.layer_dict
            return 0
        return len(self.layer_dict) - 1

    def __iter__(self):
        ''' Iterate through layer names. '''
        for layer_name in self.layer_dict.keys():
            if layer_name == self.INPUT_LAYER_KEY:
                continue
            yield layer_name

    def __getitem__(self, layer_name):
        ''' Get the layer by name. '''
        try:
            return self.layer_dict[layer_name]
        except KeyError:
            try:
                return self.ext_dict[layer_name]
            except KeyError as e:
                raise KeyError('Network: {} layer not found.'.format(str(e)))

    def __str__(self):
        str_ = 'Network: {}\n'.format(self.net_name)
        for layer_name in self:
            prevs = self.prevs(layer_name)
            prev_str = ' | '.join(['None' if n is None else n for n in prevs])
            str_ += '  Layer {} <- {}\n'.format(layer_name, prev_str)
        return str_
nn_dataflow/core/layer.py
class ConvLayer(Layer):
    '''
    NN convolutional layer parameters.

    nifm (C): # ifmap channels
    nofm (M): # ofmap channels
    hifm, wifm (H): ifmap height/width
    hofm, wofm (E): ofmap height/width
    hfil, wfil (R): weight filter width/height
    htrd, wtrd (U): stride height/width
    '''

    def __init__(self, nifm, nofm, sofm, sfil, strd=1):
        super(ConvLayer, self).__init__(nofm, sofm, strd=strd)

        if isinstance(sfil, int):
            hfil = sfil
            wfil = sfil
        elif len(sfil) == 2:
            hfil = sfil[0]
            wfil = sfil[1]
        else:
            raise ValueError('ConvLayer: sfil is invalid ({}), '
                             'needs to be either one integer or '
                             'a pair of integers'.format(sfil))

        self.hfil = hfil
        self.wfil = wfil

        hifm = self.hfil + (self.hofm - 1) * self.htrd
        wifm = self.wfil + (self.wofm - 1) * self.wtrd
        self.inlayer = Layer(nifm, (hifm, wifm))

    @staticmethod
    def data_loops():
        dls = [None] * de.NUM
        dls[de.FIL] = DataDimLoops(le.IFM, le.OFM)
        dls[de.IFM] = DataDimLoops(le.IFM, le.BAT)
        dls[de.OFM] = DataDimLoops(le.OFM, le.BAT)
        return tuple(dls)

    def input_layer(self):
        return self.inlayer

    def ops_per_neuron(self):
        # 2D convolution across all ifmap channels.
        return self.hfil * self.wfil * self.nifm

    def filter_size(self, word_size=1):
        '''
        Get size of one weight filter.

        If `word_size` is set to word byte size, return size in bytes.
        '''
        return self.hfil * self.wfil * word_size

    def total_filter_size(self, word_size=1):
        '''
        Get total size of all weight filters.

        If `word_size` is set to word byte size, return size in bytes.
        '''
        return self.nifm * self.nofm * self.filter_size(word_size)

    def __repr__(self):
        return '{}({})'.format(
            self.__class__.__name__,
            ', '.join([
                'nifm={}'.format(repr(self.nifm)),
                'nofm={}'.format(repr(self.nofm)),
                'sofm={}'.format(repr((self.hofm, self.wofm))),
                'sfil={}'.format(repr((self.hfil, self.wfil))),
                'strd={}'.format(repr((self.htrd, self.wtrd)))]))
nn_dataflow/core/layer.py
class FCLayer(ConvLayer):
    '''
    NN fully-connected layer parameters.

    As a special case of CONVLayer.

    hifm = hfil, wifm = wfil, strd = 1, hofm = wofm = 1
    '''

    def __init__(self, nifm, nofm, sfil=1):
        super(FCLayer, self).__init__(nifm, nofm, 1, sfil)
        assert self.hofm == 1 and self.wofm == 1

    def __repr__(self):
        return '{}({})'.format(
            self.__class__.__name__,
            ', '.join([
                'nifm={}'.format(repr(self.nifm)),
                'nofm={}'.format(repr(self.nofm)),
                'sfil={}'.format(repr((self.hfil, self.wfil)))]))
nn_dataflow/core/inter_layer_pipeline.py
class InterLayerPipeline():
    '''
    Inter-layer pipeline.
    '''

    def __init__(self, network, batch_size, resource, max_util_drop=0.05):
        if not isinstance(network, Network):
            raise TypeError('InterLayerPipeline: network must be '
                            'a Network instance.')
        if not isinstance(resource, Resource):
            raise TypeError('InterLayerPipeline: resource must be '
                            'a Resource instance.')
        if not 0 <= max_util_drop <= 1:
            raise ValueError('InterLayerPipeline: max_util_drop must be '
                             'between [0, 1].')

        self.network = network
        self.batch_size = batch_size
        self.resource = resource
        self.max_util_drop = max_util_drop

        self._calc_sched_dag()

        # Vertices starting from which we have generated the segments.
        self.seg_vertex_done = set()

    def ordered_layer_list(self):
        '''
        Get a list of the layers in their topological order in the scheduling
        DAG.
        '''
        return list(sum(self.dag_vertex_list, tuple()))

    def gen_segment(self, options):
        '''
        Generate all valid inter-layer pipelining segments.
        '''

        kwargs = {'network': self.network,
                  'batch_size': self.batch_size,
                  'resource': self.resource,
                  'max_util_drop': self.max_util_drop,
                  'with_opt': options.layer_pipeline_opt,
                 }

        # No pipelining, each layer sequentially occupies the whole resource.
        for layer in self.network:
            seg = ((layer,),)
            segment = PipelineSegment(seg, **kwargs)
            assert segment.valid
            yield segment

        # Pipelining.
        for vseg in self._gen_vseg():

            if len(vseg) > options.layer_pipeline_max_degree:
                continue

            if len(vseg) == 1 and len(self.dag_vertex_list[vseg[0]]) == 1:
                # An individual layer, already returned in no-pipelining case.
                continue

            # Use set to eliminate duplicates.
            seg_cands = set()

            if options.partition_interlayer:
                # Spatial pipelining.
                seg = tuple(self.dag_vertex_list[vidx] for vidx in vseg)
                seg_cands.add(seg)

            if options.hw_gbuf_save_writeback:
                # Temporal pipelining.
                # Reduce the spatial dimension.
                seg = (tuple(itertools.chain.from_iterable(
                    self.dag_vertex_list[vidx] for vidx in vseg)),)
                seg_cands.add(seg)

            # Determine segment allocation.
            for seg in seg_cands:
                segment = PipelineSegment(seg, **kwargs)
                if segment.valid:
                    yield segment

    def _gen_vseg(self, vertex_idx=0, done=None):
        '''
        Generate vertex segments starting from vertex `vertex_idx`. Yield a
        tuple of the vertices in the segment.

        `done` is a set of vertices which have already been scheduled and the
        output is already in memory.

        Rules:

        1. If a vertex does not share any dependencies with the current
        segment, i.e., none of its previous vertices is in the current segment
        or among the previous vertices of the current segment, we do not add it
        to the segment, because there is no benefit to co-locate them.

        2. If a vertex has multiple previous vertices, at most one of them
        can be in the same segment as this vertex, because the output data
        availability timing of multiple previous vertices may not match.

        3. If a vertex has multiple next vertices, either all or at most one of
        them can be NOT in the same segment as this vertex, because only
        including a small subset saves little data write-back to memory.
        '''

        vseg = tuple()

        if not done:
            done = set()
            # Reset.
            self.seg_vertex_done = set()

        if self.dag_input_vertex not in done:
            # Input layer is always in memory.
            done.add(self.dag_input_vertex)

        # The frontier is the vertex to be considered to be added to the
        # current segment.
        for frontier in range(vertex_idx, len(self.dag_vertex_list)):

            # Check whether the frontier can be added to the current segment.

            frontier_prevs = self.dag_prev_dict[frontier]

            # Whether the frontier share dependencies with the current segment,
            # if the segment is not empty.
            share_deps = not vseg or not frontier_prevs.isdisjoint(
                set.union(set(vseg), *[self.dag_prev_dict[i] for i in vseg]))

            # Whether multiple previous vertices are in the current segment.
            multi_prevs = len(frontier_prevs.intersection(vseg)) > 1

            if not share_deps or multi_prevs:
                # Not sharing any dependencies (rule 1), or multiple previous
                # vertices in the current segment (rule 2).

                # Make sure the current segment is not empty.
                assert vseg
                # Not extend the segment any more. Note that the current
                # segment has already been yielded, as well as the recursion,
                # in the last iteration.
                break

            # Extend the segment.
            vseg += (frontier,)

            # Check whether the segment is valid.

            for idx in vseg:
                nexts = self.dag_next_dict[idx]

                # The next vertices should either all or at most one not in the
                # segment (rule 3).
                if not nexts.isdisjoint(vseg) \
                        and len(nexts.difference(vseg)) > 1:
                    # The segment is invalid. Need to add more vertices.
                    break
            else:
                # The segment is valid.
                yield vseg

                # Skip if have done.
                if frontier + 1 in self.seg_vertex_done:
                    continue

                # Recursion.
                for tpl in self._gen_vseg(frontier + 1, done.union(vseg)):
                    yield tpl

        assert vertex_idx not in self.seg_vertex_done
        self.seg_vertex_done.add(vertex_idx)

    def _calc_sched_dag(self):
        '''
        Build the scheduling DAG of the network. We merge layers with no
        filters into their last previous layer, so a DAG vertex can contain one
        or more layers.

        We order and index the DAG vertices in their depth-first topological
        order. This will also be the order to schedule the layers.

        Also establish two dicts for the previous and next vertices of each DAG
        vertex.

        In summary, the attributes initialized include: `dag_input_vertex`,
        `dag_vertex_list`, `dag_vertex_dict`, `dag_prev_dict`, `dag_next_dict`.
        '''

        # Vertex of the input layer.
        self.dag_input_vertex = -1

        # The DAG vertex set. Each vertex is a merged layer tuples, represented
        # by their layer names. Use a list type to make modification easier.
        dag_vertex_set = []

        for layer_name in self.network:
            layer = self.network[layer_name]

            if isinstance(layer, ConvLayer):
                dag_vertex_set.append((layer_name,))

            else:
                prevs = set(self.network.prevs(layer_name))
                assert prevs

                # Find and merge to a vertex if that vertex only contains one
                # previous layer at the last, because non-last previous layer
                # will not have its data available to be used for this layer.
                # Also the previous layer can only have this one next layer,
                # because its data will be overwritten by this layer locally.

                # Check vertices in the reversed order.
                for idx in reversed(range(len(dag_vertex_set))):
                    vhead = dag_vertex_set[idx][:-1]
                    vtail = dag_vertex_set[idx][-1]
                    if prevs.isdisjoint(vhead) and vtail in prevs \
                            and len(self.network.nexts(vtail)) == 1:
                        dag_vertex_set[idx] += (layer_name,)
                        break
                else:
                    # No valid vertex to merge.
                    dag_vertex_set.append((layer_name,))

        assert sum(len(v) for v in dag_vertex_set) == len(self.network)

        # The DAG vertex list in the topological order.
        self.dag_vertex_list = self._topological_order(dag_vertex_set)

        # Make a directory from layer name to DAG vertex index.
        self.dag_vertex_dict = {}

        for vidx, v in enumerate(self.dag_vertex_list):
            for layer_name in v:
                assert layer_name not in self.dag_vertex_dict
                self.dag_vertex_dict[layer_name] = vidx

        # Add the input layer.
        self.dag_vertex_dict[self.network.INPUT_LAYER_KEY] = \
                self.dag_input_vertex
        # Add the external layers.
        for ext_layer in self.network.ext_layers():
            self.dag_vertex_dict[ext_layer] = self.dag_input_vertex

        # The previous and next relationship of the DAG vertices.
        self.dag_prev_dict = dict((vidx, set()) for vidx
                                  in range(len(self.dag_vertex_list)))
        self.dag_next_dict = dict((vidx, set()) for vidx
                                  in range(len(self.dag_vertex_list)))

        for layer_name in self.network:
            vidx = self.dag_vertex_dict[layer_name]

            # Previous layers.
            for p in self.network.prevs(layer_name):
                pvidx = self.dag_vertex_dict[p] \
                        if p and p not in self.network.ext_layers() \
                        else self.dag_input_vertex
                if pvidx != vidx:
                    self.dag_prev_dict[vidx].add(pvidx)

            # Next layers.
            for n in self.network.nexts(layer_name):
                if not n:
                    continue
                nvidx = self.dag_vertex_dict[n]
                if nvidx != vidx:
                    self.dag_next_dict[vidx].add(nvidx)

        # Add next layers of the input layer.
        self.dag_next_dict[self.dag_input_vertex] = set()
        for vidx in self.dag_prev_dict:
            if self.dag_input_vertex in self.dag_prev_dict[vidx]:
                self.dag_next_dict[self.dag_input_vertex].add(vidx)

    def _topological_order(self, dag_vertex_set):
        '''
        Order the DAG vertices in topological order using DFS.

        Specifically, The backtrace order of the depth-first search is the
        inverse of the topological order. See
        https://en.wikipedia.org/wiki/Topological_sorting#Depth-first_search
        '''

        # The visited layers in the DFS order.
        visited = []
        # The unseen pending layers.
        unseen = set(dag_vertex_set)
        # The layers that have been seen, but not visited due to unvisited
        # previous layers.
        seen = set()

        def _dfs(vertex):
            assert vertex not in seen
            if vertex in visited:
                return

            unseen.discard(vertex)
            seen.add(vertex)

            nexts = []
            for l in vertex:
                for n in self.network.nexts(l):
                    if n and n not in vertex and n not in nexts:
                        nexts.append(n)

            # Visit next layers in the reversed order, so the reversed visit
            # order has the original order.
            next_vertices = []
            for n in reversed(nexts):
                for nv in unseen:
                    if n in nv:
                        next_vertices.append(nv)

            for nv in next_vertices:
                _dfs(nv)

            visited.append(vertex)
            seen.remove(vertex)

        # Start from the first layers.
        start_vertices = []
        for l in reversed(self.network.firsts()):
            for v in unseen:
                if l in v:
                    start_vertices.append(v)
        for v in start_vertices:
            _dfs(v)
        assert not unseen
        assert not seen

        return list(reversed(visited))
nn_dataflow/core/layer.py
class InputLayer(Layer):
    '''
    NN input layer parameters.
    '''

    @staticmethod
    def data_loops():
        dls = [None] * de.NUM
        dls[de.FIL] = DataDimLoops()
        dls[de.IFM] = DataDimLoops()
        dls[de.OFM] = DataDimLoops(le.OFM, le.BAT)
        return tuple(dls)

    def input_layer(self):
        return None

    def ops_per_neuron(self):
        return 0
nn_dataflow/core/pipeline_segment.py
class PipelineSegment():
    '''
    Inter-layer pipeline segment.

    Segment is a two-level layer hierarchy, where the first level is spatially
    scheduled and the second level is temporally scheduled.
    '''

    # pylint: disable=too-many-instance-attributes

    # Scheduling index in the segment, as a tuple of spatial and temporal
    # scheduling indices.
    SchedIndex = namedtuple('SchedIndex', ['sp_idx', 'tm_idx'])

    def __init__(self, seg, network, batch_size, resource, max_util_drop=0.05,
                 with_opt=True):
        if not isinstance(seg, tuple):
            raise TypeError('PipelineSegment: seg must be a tuple.')
        for ltpl in seg:
            if not isinstance(ltpl, tuple):
                raise TypeError('PipelineSegment: seg must be a tuple '
                                'of sub-tuples.')

        if not isinstance(network, Network):
            raise TypeError('PipelineSegment: network must be '
                            'a Network instance.')
        if not isinstance(resource, Resource):
            raise TypeError('PipelineSegment: resource must be '
                            'a Resource instance.')

        self.seg = seg
        self.network = network
        self.batch_size = batch_size
        self.resource = resource
        self.max_util_drop = max_util_drop
        self.with_opt = with_opt

        self.valid = self._init_deps()
        if not self.valid:
            return

        # Resource allocation.
        self.valid = self._alloc_resource(max_util_drop=max_util_drop)
        if not self.valid:
            return

        # Scheduling constraints.
        self.valid = self._init_sym_cstrs()
        if not self.valid:
            return

    def allocation(self):
        '''
        Get resource allocation, as a tuple of sub-tuples corresponding to the
        layers in the segment.
        '''
        if not self.valid:
            return None
        return self.alloc

    def gen_constraint(self, max_time_overhead=float('inf')):
        '''
        Generate scheduling constraint for the segment, as a tuple of
        sub-tuples of SchedulingConstraint instances, corresponding to the
        layers in the segment.

        Yield the segment constraint tuple, and hints for pruning.

        Pruning hints are the top-level loop blocking factors. Smaller hints
        indicate better (lower) cost, and larger hints indicate better segment
        timing (with lower time overhead). Constraints with smaller hints are
        generated before those with larger hints. So if a constraint results in
        a valid scheduling, the later ones with all hints larger than its can
        be pruned.
        '''
        syms = self.cstr_symvals.keys()
        vals = self.cstr_symvals.values()
        assert syms and vals

        # Sort from small to large.
        # This is not a strict ordering, but we guarantee that if all values in
        # hint A are larger than the corresponding values in hint B, A will be
        # generated after B.
        vals = [sorted(v) for v in vals]
        syms = list(syms)

        if self.cstr_topbat_idx is not None:
            # Tovhd =  (1 + 1/to + 1 + 1/to + ...) / tb
            #       >= (1 + 1 + ...) / tb = num_sp_fbs / tb
            min_topbat = 1. * self.cstr_num_sp_fbs / max_time_overhead
            pos = self.cstr_topbat_idx
            vals[pos] = [t for t in vals[pos] if t >= min_topbat]

        for valp in itertools.product(*vals):

            constraint = tuple()

            for atpl in self._subs_symargs(self.cstr_symargs,
                                           tuple(zip(syms, valp))):
                ctpl = tuple()
                for a in atpl:
                    # Construct kwargs, adjust the types of the values.
                    kwargs = {}
                    kwargs['topbat'] = int(a.get('topbat', 0))
                    kwargs['fbifm'] = bool(a.get('fbifm', False))
                    if not kwargs['fbifm']:
                        kwargs['topifm'] = int(a.get('topifm', 0))
                    kwargs['fbofm'] = bool(a.get('fbofm', False))
                    if not kwargs['fbofm']:
                        kwargs['topofm'] = int(a.get('topofm', 0))
                    kwargs['update_dict'] = a.get('update_dict')

                    c = Cstr(**kwargs)
                    ctpl += (c,)
                constraint += (ctpl,)

            if None in valp:
                assert len(valp) == 1
                hints = (1,)
            else:
                hints = tuple(valp)

            yield constraint, hints

    def __getitem__(self, index):
        return self.seg[index]

    def __iter__(self):
        return self.seg.__iter__()

    def __len__(self):
        return len(self.seg)

    def __eq__(self, other):
        if isinstance(other, self.__class__):
            # pylint: disable=protected-access
            return self._key_attrs() == other._key_attrs()
        return NotImplemented

    def __ne__(self, other):
        return not self == other

    def __hash__(self):
        return hash(tuple(self._key_attrs()))

    def __repr__(self):
        return '{}({})'.format(
            self.__class__.__name__,
            ', '.join([
                'seg={}'.format(repr(self.seg)),
                'network={}'.format(repr(self.network)),
                'batch_size={}'.format(repr(self.batch_size)),
                'resource={}'.format(repr(self.resource)),
                'max_util_drop={}'.format(repr(self.max_util_drop)),
                'with_opt={}'.format(repr(self.with_opt))]))

    def _key_attrs(self):
        ''' Used for comparison. '''
        return (self.seg, self.network, self.batch_size, self.resource,
                self.max_util_drop, self.with_opt)

    def _init_deps(self):
        '''
        Initialize the dependency relationship of the layers in the segment as
        a mapping of the scheduling indices, and check validation. Return
        whether the segment is valid to schedule.

        We categorize dependencies to 3 categories:
        - local: with the same spatial index but different temporal indices;
        - neighbor: with different spatial indices but in the same segment;
        - memory: in different segments, from/to memory.

        The values of the src/dst dicts are tuples of indices of the neighbor
        dependencies. A layer can have at most one neighbor source (must be a
        last temporal scheduled layer), but may have multiple neighbor
        destinations (could be temporal scheduled in the middle). Also, all
        layers with the same spatial index can have at most one neighbor
        source.

        Special index `None` means memory dependency, i.e., from/to memory.
        Memory sources and neighbor sources must be mutual exclusive, in order
        to correctly set the src data regions; memory destinations and neighbor
        destinations can co-exist.

        Local dependencies are omitted, as by default each layer has its
        immediately previous layer as local source and immediately next layer
        as local destination.

        Construct an ifmap forwarding dict for shared memory source data. It
        maps previous layer name tuples, to a list of scheduling indices of all
        layers in this segment that share these exact previous layers. The
        first in the list is responsible to fetch the previous layer data and
        to forward them to others. We allow shared memory source data between
        two layers only when both layers have memory dependency only (so their
        temporal indices must be 0), and their previous layers are exactly the
        same.

        Construct an ofmap forwarding dict for multiple destinations of both
        on-chip and off-chip. It maps the scheduling index of a layer in this
        segment that has both memory and neighbor/local destinations (so needs
        to store its ofmaps back to memory), to a list of scheduling indices of
        all layers in this segment that accepts its ofmaps as ifmaps. Neighbor
        dependencies are only between the last temporal one and the first
        temporal ones; local dependencies are only between adjacent temporal
        ones.
        '''

        self.src_dict = [[None for _ in ltpl] for ltpl in self.seg]
        self.dst_dict = [[None for _ in ltpl] for ltpl in self.seg]

        self.ifm_fwd_dict = {}
        self.ofm_fwd_dict = {}

        # Mapping from layer to spatial/temporal indices in the segment.
        layer2idx = {l: PipelineSegment.SchedIndex(sp_idx, tm_idx)
                     for sp_idx, ltpl in enumerate(self.seg)
                     for tm_idx, l in enumerate(ltpl)}

        # Mapping from previous layer tuple to layer.
        prevs2layer = {}

        for sp_idx, ltpl in enumerate(self.seg):

            single_nbr_src = None

            for tm_idx, l in enumerate(ltpl):

                assert layer2idx[l] == (sp_idx, tm_idx)

                # Sources.
                src = tuple()

                prevs = self.network.prevs(l)
                assert all(p not in layer2idx or layer2idx[p] < layer2idx[l]
                           for p in prevs)
                mem_src = [p for p in prevs if p not in layer2idx]
                lcl_src = [p for p in prevs if p not in mem_src
                           and layer2idx[p].sp_idx == sp_idx]
                nbr_src = [p for p in prevs if p not in mem_src + lcl_src]

                # Ensure single local source to be the immediately previous.
                # Check at the destination so here are assertions.
                if not lcl_src:
                    assert tm_idx == 0
                else:
                    assert len(lcl_src) == 1 \
                            and layer2idx[lcl_src[0]].tm_idx == tm_idx - 1

                # Mutual exclusive.
                if mem_src and nbr_src:
                    # We now allow each spatial scheduling (vertex) to have
                    # both memory source and neighbor source when generating
                    # segments. But each single layer cannot have both;
                    # otherwise there would be multiple source data regions.
                    return False

                if mem_src:
                    # Memory source.
                    src += (None,)
                if nbr_src:
                    # Neighbor source.
                    # Single neighbor source to be the last temporal scheduled.
                    assert len(nbr_src) == 1
                    prev_idx = layer2idx[nbr_src[0]]
                    assert prev_idx.tm_idx == len(self.seg[prev_idx.sp_idx]) - 1
                    # Single neighbor source across this spatial scheduling.
                    if single_nbr_src is not None:
                        return False
                    single_nbr_src = prev_idx
                    src += (prev_idx,)

                # Shared memory source.
                if mem_src and not lcl_src:
                    assert not nbr_src
                    assert tm_idx == 0
                    if prevs in prevs2layer:
                        fet_idx = layer2idx[prevs2layer[prevs]]
                        self.ifm_fwd_dict.setdefault(prevs, [fet_idx]).append(
                            layer2idx[l])
                    else:
                        prevs2layer[prevs] = l

                # Destinations.
                dst = tuple()

                nexts = self.network.nexts(l)
                assert all(n not in layer2idx or layer2idx[n] > layer2idx[l]
                           for n in nexts)
                mem_dst = [n for n in nexts if n not in layer2idx]
                lcl_dst = [n for n in nexts if n not in mem_dst
                           and layer2idx[n].sp_idx == sp_idx]
                nbr_dst = [n for n in nexts if n not in mem_dst + lcl_dst]

                # Ensure single local destination to be the immediate next.
                if not lcl_dst:
                    if tm_idx != len(ltpl) - 1:
                        # Not utilize local data, sub-optimal.
                        return False
                else:
                    if len(lcl_dst) != 1 \
                            or layer2idx[lcl_dst[0]].tm_idx != tm_idx + 1:
                        # Local data will not be available if not adjacent.
                        return False

                # Mutual exclusive.
                # Now they can co-exist.
                # assert not mem_dst or not nbr_dst
                if mem_dst and nbr_dst:
                    assert tm_idx == len(ltpl) - 1
                    self.ofm_fwd_dict[layer2idx[l]] = [layer2idx[n]
                                                       for n in nbr_dst]
                if mem_dst and lcl_dst:
                    assert not nbr_dst
                    self.ofm_fwd_dict[layer2idx[l]] = [layer2idx[lcl_dst[0]]]

                if mem_dst:
                    # Memory destination.
                    dst += (None,)
                if nbr_dst:
                    # Neighbor destinations.
                    # This layer is the last temporal scheduled.
                    assert tm_idx == len(ltpl) - 1
                    dst += tuple(layer2idx[n] for n in nbr_dst)

                # Basic pipelining requires a linear structure (on-chip).
                if not self.with_opt:
                    if len(nbr_src) + len(lcl_src) > 1 \
                            or len(nbr_dst) + len(lcl_dst) > 1 \
                            or ((sp_idx, tm_idx) != (0, 0)
                                    and not nbr_src and not lcl_src):
                        return False

                self.src_dict[sp_idx][tm_idx] = src
                self.dst_dict[sp_idx][tm_idx] = dst

        return True

    def _alloc_resource(self, max_util_drop=0.05):
        '''
        Decide the resource allocation. Return whether the allocation succeeds.

        `max_util_drop` specifies the maximum utilization drop due to mismatch
        throughput between layers.
        '''

        self.alloc = tuple()

        # Allocate processing subregions.
        subregions = self._alloc_proc(max_util_drop=max_util_drop)
        if not subregions:
            return False

        no_time_mux = len(self.network) == sum(len(ltpl) for ltpl in self.seg)
        # All layers that have model filters must be spatially scheduled.
        if no_time_mux:
            for ltpl in self.seg:
                if len([l for l in ltpl
                        if isinstance(self.network[l], ConvLayer)]) > 1:
                    no_time_mux = False
                    break

        for sp_idx, ltpl in enumerate(self.seg):

            # Resource for the subregion.
            rtpl = tuple()

            for tm_idx, _ in enumerate(ltpl):

                # Processing region.
                proc_region = subregions[sp_idx]

                # Data source.
                src = self.src_dict[sp_idx][tm_idx]
                if None in src:
                    # Data source is memory.
                    assert src == (None,)
                    src_data_region = self.resource.src_data_region
                    for sh_idx_list in self.ifm_fwd_dict.values():
                        # Find shared memory source to use forwarding.
                        if (sp_idx, tm_idx) in sh_idx_list[1:]:
                            src_data_region = subregions[sh_idx_list[0].sp_idx]
                            break
                elif src:
                    # Data source is neighbor.
                    assert len(src) == 1
                    src_data_region = subregions[src[0].sp_idx]
                else:
                    # Data source is all local.
                    src_data_region = proc_region

                # Data destination.
                dst = self.dst_dict[sp_idx][tm_idx]
                if None in dst:
                    # Data destination is memory.
                    # assert dst == (None,)
                    # Now we can have both memory and neighbor destinations. If
                    # they co-exist, we need to store them locally and also
                    # store back to memory. In this case the dst data region is
                    # set to memory.
                    dst_data_region = self.resource.dst_data_region
                elif dst:
                    # Data destinations are neighbors.
                    # Put data in local. The next layers will fetch.
                    dst_data_region = proc_region
                else:
                    # Data destination is all local.
                    dst_data_region = proc_region

                # Make resource.
                # Note that DRAM bandwidth is not split here. We optimistically
                # assume each layer can use the full DRAM bandwidth at
                # different time. We adjust this assumption when calculating
                # the segment timing.
                rtpl += (self.resource._replace(
                    proc_region=proc_region,
                    src_data_region=src_data_region,
                    dst_data_region=dst_data_region,
                    no_time_mux=no_time_mux),)

            assert len(rtpl) == len(ltpl)
            self.alloc += (rtpl,)
        assert len(self.alloc) == len(self.seg)

        return True

    def _alloc_proc(self, max_util_drop=0.05):
        '''
        Allocate processing subregions for the segment.

        Return a list of processing subregions corresponding to the first-level
        (spatial scheduled) layers in the segment. Return None if allocation
        failed.

        `max_util_drop` specifies the maximum utilization drop due to mismatch
        throughput between layers.
        '''

        # Spatial allocation.
        proc_region = self.resource.proc_region
        dim_nodes = proc_region.dim
        total_nodes = dim_nodes.size()

        # Number of operations of each spatial allocation.
        ops = [sum(self.network[l].total_ops() for l in ltpl)
               for ltpl in self.seg]

        # Enforce a common factor among the numbers of nodes allocated to all
        # vertices in the segment. Such common factor is likely to be the
        # common height of the vertex node regions.
        common_factor_list = [cf for cf, _ in util.factorize(dim_nodes.h, 2)]

        for cf in sorted(common_factor_list, reverse=True):
            # Pick the largest common factor within the utilization constraint.

            # Number of nodes of each vertex should be approximate to the
            # number of ops of the vertex.
            nodes_raw = [o * 1. / sum(ops) * total_nodes for o in ops]

            # Round to the common factor multiples.
            assert total_nodes % cf == 0
            nodes = [max(1, int(round(nr / cf))) * cf for nr in nodes_raw]
            # Fix margin.
            while sum(nodes) != total_nodes:
                diff = [n - nr for n, nr in zip(nodes, nodes_raw)]
                if sum(nodes) > total_nodes:
                    # Decrease the nodes for the vertex with the maximum
                    # positive difference.
                    idx, _ = max(enumerate(diff), key=lambda tpl: tpl[1])
                    nodes[idx] -= cf
                else:
                    # Increase the nodes for the vertex with the minimum
                    # negative difference.
                    idx, _ = min(enumerate(diff), key=lambda tpl: tpl[1])
                    nodes[idx] += cf

            if 0 in nodes:
                continue

            # Utilization.
            time = max(o * 1. / n for o, n in zip(ops, nodes))
            utilization = sum(ops) / time / sum(nodes)
            assert utilization < 1 + 1e-6

            if utilization >= 1 - max_util_drop:
                # Found
                break

        else:
            # Not found.
            return None

        # Allocate in the processing region according to the number of nodes.
        subregions = proc_region.allocate(nodes)
        assert subregions
        assert len(subregions) == len(self.seg)
        if len(subregions) == 1:
            assert subregions[0] == proc_region

        return subregions

    def _init_sym_cstrs(self):
        '''
        Initialize the symbolic scheduling constraints for the layers in the
        segment, by constructing a nested lists of dicts `cstr_symargs` whose
        values can be symbolic expressions for the keyword arguments of layers
        in the segment, and a dict `cstr_symvals` mapping each symbol to its
        possible numerical values.

        Rules for constraints.

        - Top BAT loop factor.

        With a single layer, there is no constraint on the top BAT loop factor.
        Otherwise all layers must share the same factor, namely `topbat_shr`.

        - Fmap forwarding and fully buffering.

        Only CONV layers require to fully buffer fmaps. Local-region layers
        process data in a streaming manner.

        Each CONV layer, and all local-region layers immediately following it
        within the same spatial scheduling, are made into a group G.

        (initial) if G is both the first spatial and the first temporal
        scheduling with a CONV layer, it can choose whether to fully buffer
        ofmaps or not. This is a configuration to explore, namely `fbofm_init`.
        We decide its value by choosing the one that gives the fewer fully
        buffered inter-spatial pairs on the critical forwarding path, and the
        smaller maximum fully buffered data size.

        (within-group) within G, the CONV layer, and all local-region layers,
        should use the same top OFM factors (IFM factors are automatically
        determined by OFM factors in local-region layers), unless CONV ofmaps
        need to be fully buffered, in which case, the CONV layer and the last
        layer in G fully buffer ofmaps (top OFM factor is 1), and other layers
        still use the same top OFM factors but can be different than 1.

        (inter-temporal) if G has a source from G' in the same spatial
        scheduling (which must be immediately before G), G should fully buffer
        ifmaps, and G' should fully buffer ofmaps.

        (inter-spatial) if G has a source from G' in another spatial scheduling
        (where the source must be the last temporal scheduling in G' and that
        spatial scheduling),
        (a) if G' already fully buffers ofmaps, make G fully buffer ifmaps.
        (b) otherwise, make G fully buffer ofmaps (do not require G' to fully
            buffer ifmaps; leave it to other rules, e.g. inter-temporal, to
            decide); forward data between G' and G, by matching their top O/IFM
            factors (biasing this case for smaller pipeline filling delay).
        Notice the destination can be: (1) the leading CONV layer, whose top
        IFM factor is constrained; (2) a local-region layer, where we constrain
        the top OFM factors of this group (except otherwise constrained by
        fully buffering ofmaps).
        '''
        # pylint: disable=too-many-branches

        # Symbolic variables mapping to numerical values.
        symvals = dict()

        # Top BAT loop factor.
        topbat = symbols('topbat_shr', integer=True)
        symvals[topbat] = [t for t, _ in util.factorize(self.batch_size, 2)]

        # Whether the initial CONV layer fully buffers ofmaps.
        fbofm_init = symbols('fbofm_init')
        symvals[fbofm_init] = [False, True]

        def _layer_topofm_vals(layer_name):
            layer = self.network[layer_name]
            # We require that the total ofmap size takes at least 5% of the
            # gbuf capacity of a single node, to avoid too fine blocking.
            tmax = layer.total_ofmap_size(self.batch_size) \
                    / (0.05 * self.resource.size_gbuf)
            vals = [t for t, _ in util.factorize(layer.nofm, 2)
                    if t <= tmax or t == 1]
            assert vals
            return vals

        def _layer_topifm_vals(layer_name):
            layer = self.network[layer_name]
            # We require that the total ifmap size takes at least 5% of the
            # gbuf capacity of a single node, to avoid too fine blocking.
            tmax = layer.total_ifmap_size(self.batch_size) \
                    / (0.05 * self.resource.size_gbuf)
            vals = [t for t, _ in util.factorize(layer.nifm, 2)
                    if t <= tmax or t == 1]
            assert vals
            return vals

        # Layer constraint kwargs.
        symargs = [[{'topbat': topbat} for _ in ltpl] for ltpl in self.seg]

        # Candidates for critical forwarding path between spatial scheduling.
        sp_crit_path_cands = set()
        sp_crit_path_cands.add((0,))  # init with the first spatial.

        # The last CONV layer index.
        last_conv = PipelineSegment.SchedIndex(-1, 0)

        # Whether the current group needs to fully buffer ofmap. Delayed apply
        # to the last layer in the group.
        curr_fbofm = False

        for sp_idx, ltpl in enumerate(self.seg):

            # Initial topofm, in case of a non-CONV starting layer.
            curr_topofm = symbols('topofm_{}_s'.format(sp_idx), integer=True)
            symvals[curr_topofm] = _layer_topofm_vals(ltpl[0])

            for tm_idx, l in enumerate(ltpl):

                layer = self.network[l]
                curr_sa = symargs[sp_idx][tm_idx]

                # Neighbor source dependency.
                nsrc_sa = None
                src_deps = self.src_dict[sp_idx][tm_idx]
                if any(s is not None for s in src_deps):
                    assert len(src_deps) == 1
                    nbr_src = src_deps[0]
                    assert nbr_src.sp_idx < sp_idx
                    nsrc_sa = symargs[nbr_src.sp_idx][nbr_src.tm_idx]
                    assert nsrc_sa  # not empty, used to test nbr src exists.
                    # Set critical path candidates.
                    new_cands = set()
                    for cand in sp_crit_path_cands:
                        if cand[-1] == nbr_src.sp_idx:
                            new_cands.add(cand + (sp_idx,))
                    sp_crit_path_cands |= new_cands

                if isinstance(layer, ConvLayer):
                    # Conv layer.

                    # The last group may require to fully buffer ofmaps.
                    # Delayed apply to the immediate previous layer.
                    if curr_fbofm is not False:
                        assert last_conv >= (0, 0)
                        if last_conv.sp_idx == sp_idx:
                            assert tm_idx > 0
                            lsrc_sa = symargs[sp_idx][tm_idx - 1]
                        else:
                            lsrc_sa = symargs[last_conv.sp_idx][-1]
                        lsrc_sa['fbofm'] = curr_fbofm
                    # Reset.
                    curr_fbofm = False

                    # New topofm for a new group.
                    curr_topofm = symbols('topofm_{}_{}'.format(sp_idx, tm_idx),
                                          integer=True)
                    symvals[curr_topofm] = _layer_topofm_vals(l)

                    # Set topofm.
                    curr_sa['topofm'] = curr_topofm

                    if sp_idx == last_conv.sp_idx:
                        # Rule inter-temporal.
                        assert tm_idx > 0
                        # Make this group fully buffer ifmaps.
                        curr_sa['fbifm'] = True
                        # Make the last group fully buffer ofmaps.
                        last_sa = symargs[sp_idx][last_conv.tm_idx]
                        lsrc_sa = symargs[sp_idx][tm_idx - 1]
                        last_sa['fbofm'] = True
                        lsrc_sa['fbofm'] = True

                    elif nsrc_sa:
                        # Rule inter-spatial.
                        # We only look at this rule when inter-temporal rule
                        # does not apply and the ifmaps of this group are not
                        # yet required to fully buffer.
                        if not self.with_opt:
                            # Basic pipelining requires fully buffering all
                            # pairs of neighbor src/dst.
                            nsrc_sa['fbofm'] = True
                        nsrc_fbofm = nsrc_sa.get('fbofm', False)
                        # (a): if the source already fully buffers ofmaps.
                        # Make this group fully buffer ifmaps.
                        curr_sa['fbifm'] = symeq(nsrc_fbofm, True)
                        # (b)-(1): otherwise.
                        # Make this group fully buffer ofmaps.
                        curr_sa['fbofm'] = symeq(nsrc_fbofm, False)
                        curr_fbofm = symeq(nsrc_fbofm, False)  # delayed apply.
                        # Match top OFM/IFM factors.
                        curr_sa['topifm'] = sympiecewise(
                            (nsrc_sa['topofm'], symeq(nsrc_fbofm, False)),
                            (curr_sa.get('topifm', 0), True))

                    elif last_conv < (0, 0):
                        # The first CONV layer.
                        # Rule initial.
                        curr_sa['fbofm'] = fbofm_init
                        curr_fbofm = fbofm_init

                    last_conv = PipelineSegment.SchedIndex(sp_idx, tm_idx)

                else:
                    # Non-Conv layer.

                    if nsrc_sa:
                        # Rule inter-spatial, (b)-(2).
                        nsrc_fbofm = nsrc_sa.get('fbofm', False)
                        curr_topofm = sympiecewise(
                            (nsrc_sa['topofm'], symeq(nsrc_fbofm, False)),
                            (curr_topofm, True))
                        # Also backtrace this group.
                        for bt_idx in range(last_conv.tm_idx, tm_idx):
                            symargs[sp_idx][bt_idx]['topofm'] = curr_topofm

                    # Rule within-group.
                    curr_sa['topofm'] = curr_topofm

                # If this layer has no on-chip destinations, cancel the
                # requirement to fully buffer ofmaps.
                if all(d is None for d in self.dst_dict[sp_idx][tm_idx]) \
                        and tm_idx == len(ltpl) - 1:
                    curr_sa.pop('fbofm', False)

        # Simplify.
        self._simplify_symargs(symargs, symvals)

        # Get critical forwarding path between spatial scheduling.
        # The critical path has the longest forwarding chain.
        sp_crit_path = max(sp_crit_path_cands, key=len)

        # Check maximum fully-buffering size, and decide fbofm_init.
        opt_val = None
        opt_key = (float('inf'),) * 2  # (num of fb pairs, max fb size)
        num_sp_fbs = 0
        for val in symvals.get(fbofm_init, [False]):
            subs_symargs = self._subs_symargs(symargs, fbofm_init, val)
            maxsz = 0
            numfb = 0
            for sp_idx, (ltpl, atpl) in enumerate(zip(self.seg, subs_symargs)):
                ms = max(itertools.chain(
                    ((self.network[l].total_ofmap_size() if a.get('fbofm')
                      else 0)
                     + (self.network[l].total_ifmap_size() if a.get('fbifm')
                        else 0)
                     for l, a in zip(ltpl, atpl)),
                    [0]))  # safe max with default.
                if ms > self.alloc[sp_idx][0].proc_region.dim.size() \
                        * self.alloc[sp_idx][0].size_gbuf:
                    break
                maxsz = max(maxsz, ms)
                if sp_idx in sp_crit_path and atpl[-1].get('fbofm', False):
                    numfb += 1
            else:
                key = (numfb, maxsz)
                if key < opt_key:
                    opt_val, opt_key = val, key
                    num_sp_fbs = numfb
        if opt_val is None:
            return False
        # Use the optimal value.
        symvals[fbofm_init] = [opt_val]
        self._simplify_symargs(symargs, symvals)

        # Shared memory source must have the same topifm.
        for sh_idx_list in self.ifm_fwd_dict.values():
            assert len(sh_idx_list) > 1
            fet_sp_idx = sh_idx_list[0].sp_idx
            sh_symarg_list = [symargs[idx.sp_idx][0] for idx in sh_idx_list]

            # Must have no constraint on ifmaps access from memory.
            assert all(not sa.get('fbifm', False) and not sa.get('topifm', 0)
                       for sa in sh_symarg_list)

            # Cannot constrain both topifm and topofm.
            if any(sa.get('fbofm', False) or sa.get('topofm', 0)
                   for sa in sh_symarg_list):
                sh_kwargs = {'fbifm': True}
            else:
                topifm = symbols('topifm_{}'.format(fet_sp_idx), integer=True)
                symvals[topifm] = _layer_topifm_vals(self.seg[fet_sp_idx][0])
                sh_kwargs = {'topifm': topifm}

            # Set constraints.
            for sa in sh_symarg_list:
                sa.update(sh_kwargs)

        # Simplify.
        self._simplify_symargs(symargs, symvals)

        # Turn constraints into lazily updated rules.
        self._lazify_topofm_symargs(symargs, symvals)
        # Cannot simplify any more as update_dict is not sympifi-able.

        # Sort symbol dict.
        symvals = OrderedDict(sorted(((s, symvals[s]) for s in symvals),
                                     key=lambda item: str(item[0])))

        if not symvals:
            # Must add a dummy symbol so iterative substitution can happen.
            symvals[symbols('_dummy')] = [None]

        self.cstr_symargs = symargs
        self.cstr_symvals = symvals
        self.cstr_num_sp_fbs = num_sp_fbs
        try:
            self.cstr_topbat_idx = list(symvals.keys()).index(topbat)
        except ValueError:
            self.cstr_topbat_idx = None

        return True

    @staticmethod
    def _simplify_symargs_one_pass(symargs, symvals):
        '''
        Simplify symargs and symvals in-place:
        - If fbi/ofm is False, then remove it.
        - If fbi/ofm is True, then remove topi/ofm.
        - If a symbol can take only one value, then substitute it.
        - If a symbol only occurs once, then remove its constraint.

        Return whether the symargs and symvals are already simplified.
        '''
        for a in itertools.chain.from_iterable(symargs):
            is_fbifm = a.get('fbifm')
            is_fbofm = a.get('fbofm')
            # pylint: disable=singleton-comparison
            # lhs may be symbolic, see
            # docs.sympy.org/latest/modules/logic.html#sympy.logic.boolalg.BooleanTrue
            if is_fbifm == True:
                a.pop('topifm', 0)
            if is_fbifm == False:
                a.pop('fbifm', False)
            if is_fbofm == True:
                a.pop('topofm', 0)
            if is_fbofm == False:
                a.pop('fbofm', False)

        subs_dict = {}

        # Possible values for symbols.
        subs_dict.update(
            (s, symvals[s][0]) for s in symvals if len(symvals[s]) == 1)

        # Count the occurrence of symbols in all args (values).
        symcnts = Counter(
            s for a in itertools.chain.from_iterable(symargs)
            for val in a.values() for s in symtuple(val).free_symbols)
        assert set(symcnts.keys()).issubset(symvals.keys())
        subs_dict.update((s, None)
                         for s in set(symvals.keys()) - set(symcnts.keys()))
        subs_dict.update((s, 0 if str(s).startswith('top') else False)
                         for s in symcnts if symcnts[s] <= 1)

        # Substitute symbols and remove from symbol dict.
        for a in itertools.chain.from_iterable(symargs):
            for k in a:
                a[k] = symtuple(a[k]).subs(subs_dict)[0]
        for s in subs_dict:
            del symvals[s]

        return not subs_dict

    def _simplify_symargs(self, symargs, symvals):
        ''' Simplify symargs and symvals in-place iteratively. '''
        while not self._simplify_symargs_one_pass(symargs, symvals):
            pass
        used_syms = symtuple(
            *[symtuple(*a.values())
              for a in itertools.chain.from_iterable(symargs)]).free_symbols
        assert set(used_syms) == set(symvals.keys())
        assert all(val for val in symvals.values())

    @staticmethod
    def _subs_symargs(symargs, *subs_args):
        '''
        Substitute symbols. The additional arguments are passed to subs().

        Return a new substituted copy without modifying the original one.
        '''
        # sympify=False is necessary because there may be str in the values.
        return [[dict((k, symtuple(a[k], sympify=False).subs(*subs_args)[0])
                      for k in a) for a in atpl] for atpl in symargs]

    class TopOfmUpdateLambda(symbasic):
        ''' A sympifi-able lambda function to lazily update topofm. '''
        # pylint: disable=no-init
        def __new__(cls, *args):
            return super(PipelineSegment.TopOfmUpdateLambda, cls).__new__(cls)
        def __call__(self, arg_s, arg_r):
            setattr(arg_s, 'topofm', arg_r.scheme['to'][0])

    def _lazify_topofm_symargs(self, symargs, symvals):
        '''
        Turn qualified topofm constraints into lazily updated rules.

        If a symbol is only used as the topofm constraint by a single CONV
        layer and some local-region layers, we can turn it into a lazily update
        rule.
        '''
        sym2conv = {}  # symbol --> the only CONV layer using it.
        sym2lrs = {}   # symbol --> list of local-region layer using it.
        unqual_syms = set()  # symbols used by two or more CONV layers.
        for l, a in zip(itertools.chain.from_iterable(self.seg),
                        itertools.chain.from_iterable(symargs)):
            layer = self.network[l]
            if isinstance(layer, ConvLayer):
                topofm = a.get('topofm', 0)
                topifm = a.get('topifm', 0)
                for s in symtuple(topofm, topifm).free_symbols:
                    if s not in unqual_syms:
                        if s in sym2conv:
                            # If a symbol is used in two CONV layers, it cannot
                            # be lazily updated.
                            del sym2conv[s]
                            sym2lrs.pop(s, [])
                            unqual_syms.add(s)
                        elif topofm == s:
                            assert s not in sym2lrs
                            sym2conv[s] = l
            else:
                topofm = a.get('topofm', 0)
                if topofm in sym2conv:
                    sym2lrs.setdefault(topofm, []).append(l)
        assert 0 not in sym2conv and 0 not in sym2lrs

        syms = sym2conv.keys()  # symbols to be lazily updated.
        lr2conv = {}  # local-region layer to the CONV layer constraining it.
        for s in syms:
            for lr in sym2lrs.get(s, []):
                lr2conv[lr] = sym2conv[s]
        lconvs = set(lr2conv.values())  # CONV layer whose topofm to be removed.

        for l, a in zip(itertools.chain.from_iterable(self.seg),
                        itertools.chain.from_iterable(symargs)):
            if l in lconvs:
                # Remove CONV topofm.
                assert sym2conv[a['topofm']] == l
                del a['topofm']
            elif l in lr2conv:
                # Link local-region layer to the CONV layer.
                lconv = lr2conv[l]
                assert sym2conv[a['topofm']] == lconv
                del a['topofm']
                a['update_dict'] = {
                    lconv: PipelineSegment.TopOfmUpdateLambda()}

        for s in syms:
            del symvals[s]
nn_dataflow/core/option.py
class Option(namedtuple('Option', OPTION_LIST)):
    '''
    Schedule options.
    '''

    def __new__(cls, *args, **kwargs):

        if len(args) > len(OPTION_LIST):
            raise TypeError('Option: can take at most {} arguments ({} given).'
                            .format(len(OPTION_LIST), len(args)))

        if not set(kwargs).issubset(OPTION_LIST):
            raise TypeError('Option: got an unexpected keyword argument {}.'
                            .format(next(k for k in kwargs
                                         if k not in OPTION_LIST)))

        # Combine args and kwargs.
        kwdict = kwargs.copy()
        for k, v in zip(OPTION_LIST, args):
            if k in kwdict:
                raise TypeError('Option: got multiple values for '
                                'keyword argument {}.'
                                .format(k))
            kwdict[k] = v

        kwdict.setdefault('sw_gbuf_bypass', (False,) * de.NUM)
        kwdict.setdefault('sw_solve_loopblocking', False)
        kwdict.setdefault('hw_access_forwarding', False)
        kwdict.setdefault('hw_gbuf_sharing', False)
        kwdict.setdefault('hw_gbuf_save_writeback', False)
        kwdict.setdefault('partition_hybrid', False)
        kwdict.setdefault('partition_batch', False)
        kwdict.setdefault('partition_ifmaps', False)
        kwdict.setdefault('partition_interlayer', False)
        kwdict.setdefault('layer_pipeline_time_ovhd', float('inf'))
        kwdict.setdefault('layer_pipeline_max_degree', float('inf'))
        kwdict.setdefault('layer_pipeline_opt', True)
        kwdict.setdefault('opt_goal', 'e')
        kwdict.setdefault('ntops', 1)
        kwdict.setdefault('nprocesses', 1)
        kwdict.setdefault('verbose', False)

        assert set(kwdict) == set(OPTION_LIST)

        ntp = super(Option, cls).__new__(cls, **kwdict)

        if not isinstance(ntp.sw_gbuf_bypass, tuple):
            raise TypeError('Option: sw_gbuf_bypass must be a tuple')
        if len(ntp.sw_gbuf_bypass) != de.NUM:
            raise ValueError('Option: sw_gbuf_bypass must have length {}'
                             .format(de.NUM))

        if ntp.sw_solve_loopblocking and ntp.hw_gbuf_sharing:
            raise ValueError('Option: sw_solve_loopblocking and '
                             'hw_gbuf_sharing cannot be simultaneously '
                             'enabled.')

        if ntp.hw_access_forwarding and ntp.hw_gbuf_sharing:
            raise ValueError('Option: hw_access_forwarding is implied by '
                             'hw_gbuf_sharing, thus cannot be both enabled.')

        if ntp.sw_solve_loopblocking and ntp.hw_gbuf_save_writeback:
            raise ValueError('Option: sw_solve_loopblocking and '
                             'hw_gbuf_save_writeback cannot be simultaneously '
                             'enabled.')

        if ntp.partition_ifmaps and not ntp.partition_hybrid:
            raise ValueError('Option: partition_ifmaps requires '
                             'partition_hybrid to be set.')

        if not isinstance(ntp.layer_pipeline_time_ovhd, (int, float)):
            raise KeyError('Option: layer_pipeline_time_ovhd must be a '
                           'number.')
        if ntp.layer_pipeline_time_ovhd < 0:
            raise ValueError('Option: layer_pipeline_time_ovhd must be '
                             'positive.')

        if not isinstance(ntp.layer_pipeline_max_degree, (int, float)):
            raise KeyError('Option: layer_pipeline_max_degree must be a '
                           'number.')
        if ntp.layer_pipeline_max_degree < 0:
            raise ValueError('Option: layer_pipeline_max_degree must be '
                             'positive.')

        if ntp.opt_goal not in ['e', 'd', 'ed']:
            raise ValueError('Option: opt_goal is invalid, must be one of '
                             '\'e\', \'d\', and \'ed\'.')

        return ntp

    @staticmethod
    def option_list():
        ''' List of options. '''
        return OPTION_LIST
nn_dataflow/core/layer.py
class PoolingLayer(LocalRegionLayer):
    '''
    NN pooling layer parameters.

    As a special case of LocalRegionLayer.

    nreg = ntrd = 1
    '''

    def __init__(self, nofm, sofm, sreg, strd=None):
        if strd is None:
            strd = sreg
        super(PoolingLayer, self).__init__(nofm, sofm, 1, sreg,
                                           ntrd=1, strd=strd)
        assert self.nreg == 1
        assert self.ntrd == 1

    def __repr__(self):
        return '{}({})'.format(
            self.__class__.__name__,
            ', '.join([
                'nofm={}'.format(repr(self.nofm)),
                'sofm={}'.format(repr((self.hofm, self.wofm))),
                'sreg={}'.format(repr((self.hreg, self.wreg))),
                'strd={}'.format(repr((self.htrd, self.wtrd)))]))
nn_dataflow/core/phy_dim2.py
class PhyDim2(namedtuple('PhyDim2', ['h', 'w'])):
    '''
    Denote a physical 2D dimension.
    '''

    def size(self):
        ''' Total size. '''
        return int(reduce(mul, self, 1))

    def hop_dist(self, other):
        ''' Hop distance between twn coordinate. '''
        if not isinstance(other, PhyDim2):
            raise TypeError('PhyDim2: hop_dist only applies on two PhyDim2 '
                            'instances.')
        return abs(self.h - other.h) + abs(self.w - other.w)

    def __add__(self, other):
        ''' Return element-wise `self + other`. '''
        if not isinstance(other, PhyDim2):
            other = PhyDim2(other, other)
        return PhyDim2(*map(add, self, other))

    def __sub__(self, other):
        ''' Return element-wise `self - other`. '''
        if not isinstance(other, PhyDim2):
            other = PhyDim2(other, other)
        return PhyDim2(*map(sub, self, other))

    def __neg__(self):
        ''' Return element-wise negative. '''
        return PhyDim2(*map(neg, self))

    def __mul__(self, other):
        ''' Return element-wise `self * other`. '''
        if not isinstance(other, PhyDim2):
            other = PhyDim2(other, other)
        return PhyDim2(*map(mul, self, other))

    __rmul__ = __mul__


Question: import re
from nn_dataflow.core import InputLayer, ConvLayer, FCLayer, PoolingLayer
from nn_dataflow.core import InterLayerPipeline
from nn_dataflow.core import Network
from nn_dataflow.core import Option
from nn_dataflow.core import PhyDim2
from nn_dataflow.core import PipelineSegment
from . import TestPipelineFixture
""" $lic$
Copyright (C) 2016-2020 by Tsinghua University and The Board of Trustees of
Stanford University

This program is free software: you can redistribute it and/or modify it under
the terms of the Modified BSD-3 License as published by the Open Source
Initiative.

This program is distributed in the hope that it will be useful, but WITHOUT ANY
WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
PARTICULAR PURPOSE. See the BSD-3 License for more details.

You should have received a copy of the Modified BSD-3 License along with this
program. If not, see <https://opensource.org/licenses/BSD-3-Clause>.
"""




class TestInterLayerPipeline(TestPipelineFixture):
    ''' Tests for InterLayerPipeline. '''

    def test_valid_args(self):
        ''' Valid arguments. '''

Answer:         ilp = InterLayerPipeline(self.net['net1'], self.batch_size,