# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
import math
import tqdm
import numpy as np
from multiprocessing.pool import ThreadPool
import paddle.fluid as fluid
import paddlex.utils.logging as logging
import paddlex
import copy
import os.path as osp
from paddlex.cv.transforms import arrange_transforms
from collections import OrderedDict
from .faster_rcnn import FasterRCNN
from .utils.detection_eval import eval_results, bbox2out, mask2out

class MaskRCNN(FasterRCNN):
    """构建MaskRCNN，并实现其训练、评估、预测和模型导出。

    Args:
        num_classes (int): 包含了背景类的类别数。默认为81。
        backbone (str): MaskRCNN的backbone网络，取值范围为['ResNet18', 'ResNet50',
            'ResNet50_vd', 'ResNet101', 'ResNet101_vd', 'HRNet_W18']。默认为'ResNet50'。
        with_fpn (bool): 是否使用FPN结构。默认为True。
        aspect_ratios (list): 生成anchor高宽比的可选值。默认为[0.5, 1.0, 2.0]。
        anchor_sizes (list): 生成anchor大小的可选值。默认为[32, 64, 128, 256, 512]。
        input_channel (int): 输入图像的通道数量。默认为3。
    """

    def __init__(self,
                 num_classes=81,
                 backbone='ResNet50',
                 with_fpn=True,
                 aspect_ratios=[0.5, 1.0, 2.0],
                 anchor_sizes=[32, 64, 128, 256, 512],
                 input_channel=3):
        self.init_params = locals()
        backbones = [
            'ResNet18', 'ResNet50', 'ResNet50_vd', 'ResNet101', 'ResNet101_vd',
            'HRNet_W18'
        ]
        assert backbone in backbones, "backbone should be one of {}".format(
            backbones)
        super(FasterRCNN, self).__init__('detector')
        self.backbone = backbone
        self.num_classes = num_classes
        self.with_fpn = with_fpn
        self.anchor_sizes = anchor_sizes
        self.labels = None
        if with_fpn:
            self.mask_head_resolution = 28
        else:
            self.mask_head_resolution = 14
        self.fixed_input_shape = None
        self.input_channel = input_channel
        self.with_dcn = False

    def build_net(self, mode='train'):
        train_pre_nms_top_n = 2000 if self.with_fpn else 12000
        test_pre_nms_top_n = 1000 if self.with_fpn else 6000
        num_convs = 4 if self.with_fpn else 0
        model = paddlex.cv.nets.detection.MaskRCNN(
            backbone=self._get_backbone(self.backbone),
            num_classes=self.num_classes,
            mode=mode,
            with_fpn=self.with_fpn,
            train_pre_nms_top_n=train_pre_nms_top_n,
            test_pre_nms_top_n=test_pre_nms_top_n,
            num_convs=num_convs,
            mask_head_resolution=self.mask_head_resolution,
            fixed_input_shape=self.fixed_input_shape,
            input_channel=self.input_channel)
        inputs = model.generate_inputs()
        if mode == 'train':
            model_out = model.build_net(inputs)
            loss = model_out['loss']
            self.optimizer.minimize(loss)
            outputs = OrderedDict(
                [('loss', model_out['loss']),
                 ('loss_cls', model_out['loss_cls']),
                 ('loss_bbox', model_out['loss_bbox']),
                 ('loss_mask', model_out['loss_mask']),
                 ('loss_rpn_cls', model_out['loss_rpn_cls']), (
                     'loss_rpn_bbox', model_out['loss_rpn_bbox'])])
        else:
            outputs = model.build_net(inputs)
        return inputs, outputs

    def default_optimizer(self, learning_rate, warmup_steps, warmup_start_lr,
                          lr_decay_epochs, lr_decay_gamma,
                          num_steps_each_epoch):
        if warmup_steps > lr_decay_epochs[0] * num_steps_each_epoch:
            logging.error(
                "In function train(), parameters should satisfy: warmup_steps <= lr_decay_epochs[0]*num_samples_in_train_dataset",
                exit=False)
            logging.error(
                "See this doc for more information: https://github.com/PaddlePaddle/PaddleX/blob/develop/docs/appendix/parameters.md#notice",
                exit=False)
            logging.error(
                "warmup_steps should less than {} or lr_decay_epochs[0] greater than {}, please modify 'lr_decay_epochs' or 'warmup_steps' in train function".
                format(lr_decay_epochs[0] * num_steps_each_epoch, warmup_steps
                       // num_steps_each_epoch))
        boundaries = [b * num_steps_each_epoch for b in lr_decay_epochs]
        values = [(lr_decay_gamma**i) * learning_rate
                  for i in range(len(lr_decay_epochs) + 1)]
        lr_decay = fluid.layers.piecewise_decay(
            boundaries=boundaries, values=values)
        lr_warmup = fluid.layers.linear_lr_warmup(
            learning_rate=lr_decay,
            warmup_steps=warmup_steps,
            start_lr=warmup_start_lr,
            end_lr=learning_rate)
        optimizer = fluid.optimizer.Momentum(
            learning_rate=lr_warmup,
            momentum=0.9,
            regularization=fluid.regularizer.L2Decay(1e-04))
        return optimizer

    def train(self,
              num_epochs,
              train_dataset,
              train_batch_size=1,
              eval_dataset=None,
              save_interval_epochs=1,
              log_interval_steps=2,
              save_dir='output',
              pretrain_weights='IMAGENET',
              optimizer=None,
              learning_rate=1.0 / 800,
              warmup_steps=500,
              warmup_start_lr=1.0 / 2400,
              lr_decay_epochs=[8, 11],
              lr_decay_gamma=0.1,
              metric=None,
              use_vdl=False,
              early_stop=False,
              early_stop_patience=5,
              resume_checkpoint=None):
        """训练。

        Args:
            num_epochs (int): 训练迭代轮数。
            train_dataset (paddlex.datasets): 训练数据读取器。
            train_batch_size (int): 训练或验证数据batch大小。目前检测仅支持单卡评估，训练数据batch大小与
                显卡数量之商为验证数据batch大小。默认值为1。
            eval_dataset (paddlex.datasets): 验证数据读取器。
            save_interval_epochs (int): 模型保存间隔（单位：迭代轮数）。默认为1。
            log_interval_steps (int): 训练日志输出间隔（单位：迭代次数）。默认为20。
            save_dir (str): 模型保存路径。默认值为'output'。
            pretrain_weights (str): 若指定为路径时，则加载路径下预训练模型；若为字符串'IMAGENET'，
                则自动下载在ImageNet图片数据上预训练的模型权重；若为字符串'COCO'，
                则自动下载在COCO数据集上预训练的模型权重；若为None，则不使用预训练模型。默认为None。
            optimizer (paddle.fluid.optimizer): 优化器。当该参数为None时，使用默认优化器：
                fluid.layers.piecewise_decay衰减策略，fluid.optimizer.Momentum优化方法。
            learning_rate (float): 默认优化器的学习率。默认为1.0/800。
            warmup_steps (int):  默认优化器进行warmup过程的步数。默认为500。
            warmup_start_lr (int): 默认优化器warmup的起始学习率。默认为1.0/2400。
            lr_decay_epochs (list): 默认优化器的学习率衰减轮数。默认为[8, 11]。
            lr_decay_gamma (float): 默认优化器的学习率衰减率。默认为0.1。
            metric (bool): 训练过程中评估的方式，取值范围为['COCO', 'VOC']。
            use_vdl (bool): 是否使用VisualDL进行可视化。默认值为False。
            early_stop (bool): 是否使用提前终止训练策略。默认值为False。
            early_stop_patience (int): 当使用提前终止训练策略时，如果验证集精度在`early_stop_patience`个epoch内
                连续下降或持平，则终止训练。默认值为5。
            resume_checkpoint (str): 恢复训练时指定上次训练保存的模型路径。若为None，则不会恢复训练。默认值为None。

        Raises:
            ValueError: 评估类型不在指定列表中。
            ValueError: 模型从inference model进行加载。
        """
        if metric is None:
            if isinstance(train_dataset, paddlex.datasets.CocoDetection) or \
                    isinstance(train_dataset, paddlex.datasets.EasyDataDet):
                metric = 'COCO'
            else:
                raise Exception(
                    "train_dataset should be datasets.COCODetection or datasets.EasyDataDet."
                )
        assert metric in ['COCO', 'VOC'], "Metric only support 'VOC' or 'COCO'"
        self.metric = metric
        if not self.trainable:
            raise Exception("Model is not trainable from load_model method.")
        self.labels = copy.deepcopy(train_dataset.labels)
        self.labels.insert(0, 'background')
        # 构建训练网络
        if optimizer is None:
            # 构建默认的优化策略
            num_steps_each_epoch = train_dataset.num_samples // train_batch_size
            optimizer = self.default_optimizer(
                learning_rate=learning_rate,
                warmup_steps=warmup_steps,
                warmup_start_lr=warmup_start_lr,
                lr_decay_epochs=lr_decay_epochs,
                lr_decay_gamma=lr_decay_gamma,
                num_steps_each_epoch=num_steps_each_epoch)
        self.optimizer = optimizer
        # 构建训练、验证、测试网络
        self.build_program()
        fuse_bn = True
        if self.with_fpn and self.backbone in [
                'ResNet18', 'ResNet50', 'HRNet_W18'
        ]:
            fuse_bn = False
        self.net_initialize(
            startup_prog=fluid.default_startup_program(),
            pretrain_weights=pretrain_weights,
            fuse_bn=fuse_bn,
            save_dir=save_dir,
            resume_checkpoint=resume_checkpoint)
        # 训练
        self.train_loop(
            num_epochs=num_epochs,
            train_dataset=train_dataset,
            train_batch_size=train_batch_size,
            eval_dataset=eval_dataset,
            save_interval_epochs=save_interval_epochs,
            log_interval_steps=log_interval_steps,
            save_dir=save_dir,
            use_vdl=use_vdl,
            early_stop=early_stop,
            early_stop_patience=early_stop_patience)

    def evaluate(self,
                 eval_dataset,
                 batch_size=1,
                 epoch_id=None,
                 metric=None,
                 return_details=False):
        """评估。

        Args:
            eval_dataset (paddlex.datasets): 验证数据读取器。
            batch_size (int): 验证数据批大小。默认为1。当前只支持设置为1。
            epoch_id (int): 当前评估模型所在的训练轮数。
            metric (bool): 训练过程中评估的方式，取值范围为['COCO', 'VOC']。默认为None，
                根据用户传入的Dataset自动选择，如为VOCDetection，则metric为'VOC';
                如为COCODetection，则metric为'COCO'。
            return_details (bool): 是否返回详细信息。默认值为False。

        Returns:
            tuple (metrics, eval_details) /dict (metrics): 当return_details为True时，返回(metrics, eval_details)，
                当return_details为False时，返回metrics。metrics为dict，包含关键字：'bbox_mmap'和'segm_mmap'
                或者’bbox_map‘和'segm_map'，分别表示预测框和分割区域平均准确率平均值在
                各个IoU阈值下的结果取平均值的结果（mmAP）、平均准确率平均值（mAP）。eval_details为dict，
                包含bbox、mask和gt三个关键字。其中关键字bbox的键值是一个列表，列表中每个元素代表一个预测结果，
                一个预测结果是一个由图像id，预测框类别id, 预测框坐标，预测框得分组成的列表。
                关键字mask的键值是一个列表，列表中每个元素代表各预测框内物体的分割结果，分割结果由图像id、
                预测框类别id、表示预测框内各像素点是否属于物体的二值图、预测框得分。
                而关键字gt的键值是真实标注框的相关信息。
        """
        input_channel = getattr(self, 'input_channel', 3)
        arrange_transforms(
            model_type=self.model_type,
            class_name=self.__class__.__name__,
            transforms=eval_dataset.transforms,
            mode='eval',
            input_channel=input_channel)
        if metric is None:
            if hasattr(self, 'metric') and self.metric is not None:
                metric = self.metric
            else:
                if isinstance(eval_dataset, paddlex.datasets.CocoDetection):
                    metric = 'COCO'
                else:
                    raise Exception(
                        "eval_dataset should be datasets.COCODetection.")
        assert metric in ['COCO', 'VOC'], "Metric only support 'VOC' or 'COCO'"
        if batch_size > 1:
            batch_size = 1
            logging.warning(
                "Mask RCNN supports batch_size=1 only during evaluating, so batch_size is forced to be set to 1."
            )
        data_generator = eval_dataset.generator(
            batch_size=batch_size, drop_last=False)

        total_steps = math.ceil(eval_dataset.num_samples * 1.0 / batch_size)
        results = list()
        logging.info(
            "Start to evaluating(total_samples={}, total_steps={})...".format(
                eval_dataset.num_samples, total_steps))
        for step, data in tqdm.tqdm(
                enumerate(data_generator()), total=total_steps):
            images = np.array([d[0] for d in data]).astype('float32')
            im_infos = np.array([d[1] for d in data]).astype('float32')
            im_shapes = np.array([d[3] for d in data]).astype('float32')
            feed_data = {
                'image': images,
                'im_info': im_infos,
                'im_shape': im_shapes,
            }
            with fluid.scope_guard(self.scope):
                outputs = self.exe.run(
                    self.test_prog,
                    feed=[feed_data],
                    fetch_list=list(self.test_outputs.values()),
                    return_numpy=False)
            res = {
                'bbox': (np.array(outputs[0]),
                         outputs[0].recursive_sequence_lengths()),
                'mask': (np.array(outputs[1]),
                         outputs[1].recursive_sequence_lengths())
            }
            res_im_id = [d[2] for d in data]
            res['im_info'] = (im_infos, [])
            res['im_shape'] = (im_shapes, [])
            res['im_id'] = (np.array(res_im_id), [])
            results.append(res)
            logging.debug("[EVAL] Epoch={}, Step={}/{}".format(epoch_id, step +
                                                               1, total_steps))

        ap_stats, eval_details = eval_results(
            results,
            'COCO',
            eval_dataset.coco_gt,
            with_background=True,
            resolution=self.mask_head_resolution)
        if metric == 'VOC':
            if isinstance(ap_stats[0], np.ndarray) and isinstance(ap_stats[1],
                                                                  np.ndarray):
                metrics = OrderedDict(
                    zip(['bbox_map', 'segm_map'],
                        [ap_stats[0][1], ap_stats[1][1]]))
            else:
                metrics = OrderedDict(
                    zip(['bbox_map', 'segm_map'], [0.0, 0.0]))
        elif metric == 'COCO':
            if isinstance(ap_stats[0], np.ndarray) and isinstance(ap_stats[1],
                                                                  np.ndarray):
                metrics = OrderedDict(
                    zip(['bbox_mmap', 'segm_mmap'],
                        [ap_stats[0][0], ap_stats[1][0]]))
            else:
                metrics = OrderedDict(
                    zip(['bbox_mmap', 'segm_mmap'], [0.0, 0.0]))
        if return_details:
            return metrics, eval_details
        return metrics

    @staticmethod
    def _postprocess(res, batch_size, num_classes, mask_head_resolution,
                     labels):
        clsid2catid = dict({i: i for i in range(num_classes)})
        xywh_results = bbox2out([res], clsid2catid)
        segm_results = mask2out([res], clsid2catid, mask_head_resolution)
        preds = [[] for i in range(batch_size)]
        import pycocotools.mask as mask_util
        for index, xywh_res in enumerate(xywh_results):
            image_id = xywh_res['image_id']
            del xywh_res['image_id']
            xywh_res['mask'] = mask_util.decode(segm_results[index][
                'segmentation'])
            xywh_res['category'] = labels[xywh_res['category_id']]
            preds[image_id].append(xywh_res)

        return preds

    def predict(self, img_file, transforms=None):
        """预测。

        Args:
            img_file(str|np.ndarray): 预测图像路径，或者是解码后的排列格式为（H, W, C）且类型为float32且为BGR格式的数组。
            transforms (paddlex.det.transforms): 数据预处理操作。

        Returns:
            lict: 预测结果列表，每个预测结果由预测框类别标签、预测框类别名称、
                  预测框坐标(坐标格式为[xmin, ymin, w, h]）、
                  原图大小的预测二值图（1表示预测框类别，0表示背景类）、
                  预测框得分组成。
        """
        if transforms is None and not hasattr(self, 'test_transforms'):
            raise Exception("transforms need to be defined, now is None.")
        if isinstance(img_file, (str, np.ndarray)):
            images = [img_file]
        else:
            raise Exception("img_file must be str/np.ndarray")

        if transforms is None:
            transforms = self.test_transforms
        input_channel = getattr(self, 'input_channel', 3)
        im, im_resize_info, im_shape = FasterRCNN._preprocess(
            images,
            transforms,
            self.model_type,
            self.__class__.__name__,
            input_channel=input_channel)

        with fluid.scope_guard(self.scope):
            result = self.exe.run(self.test_prog,
                                  feed={
                                      'image': im,
                                      'im_info': im_resize_info,
                                      'im_shape': im_shape
                                  },
                                  fetch_list=list(self.test_outputs.values()),
                                  return_numpy=False,
                                  use_program_cache=True)

        res = {
            k: (np.array(v), v.recursive_sequence_lengths())
            for k, v in zip(list(self.test_outputs.keys()), result)
        }
        res['im_id'] = (np.array(
            [[i] for i in range(len(images))]).astype('int32'), [])
        res['im_shape'] = (np.array(im_shape), [])
        preds = MaskRCNN._postprocess(res,
                                      len(images), self.num_classes,
                                      self.mask_head_resolution, self.labels)

        return preds[0]

    def batch_predict(self, img_file_list, transforms=None):
        """预测。

        Args:
            img_file_list(list|tuple): 对列表（或元组）中的图像同时进行预测，列表中的元素可以是图像路径
                也可以是解码后的排列格式为（H，W，C）且类型为float32且为BGR格式的数组。
            transforms (paddlex.det.transforms): 数据预处理操作。
        Returns:
            dict: 每个元素都为列表，表示各图像的预测结果。在各图像的预测结果列表中，每个预测结果由预测框类别标签、预测框类别名称、
                  预测框坐标(坐标格式为[xmin, ymin, w, h]）、
                  原图大小的预测二值图（1表示预测框类别，0表示背景类）、
                  预测框得分组成。
        """
        if transforms is None and not hasattr(self, 'test_transforms'):
            raise Exception("transforms need to be defined, now is None.")

        if not isinstance(img_file_list, (list, tuple)):
            raise Exception("im_file must be list/tuple")

        if transforms is None:
            transforms = self.test_transforms
        input_channel = getattr(self, 'input_channel', 3)
        im, im_resize_info, im_shape = FasterRCNN._preprocess(
            img_file_list,
            transforms,
            self.model_type,
            self.__class__.__name__,
            self.thread_pool,
            input_channel=input_channel)

        with fluid.scope_guard(self.scope):
            result = self.exe.run(self.test_prog,
                                  feed={
                                      'image': im,
                                      'im_info': im_resize_info,
                                      'im_shape': im_shape
                                  },
                                  fetch_list=list(self.test_outputs.values()),
                                  return_numpy=False,
                                  use_program_cache=True)

        res = {
            k: (np.array(v), v.recursive_sequence_lengths())
            for k, v in zip(list(self.test_outputs.keys()), result)
        }
        res['im_id'] = (np.array(
            [[i] for i in range(len(img_file_list))]).astype('int32'), [])
        res['im_shape'] = (np.array(im_shape), [])
        preds = MaskRCNN._postprocess(res,
                                      len(img_file_list), self.num_classes,
                                      self.mask_head_resolution, self.labels)
        return preds


#!/usr/bin/env python3
# Copyright (c) 2014-2016 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""Test the BIP66 changeover logic."""

from test_framework.test_framework import eBitcashTestFramework
from test_framework.util import *

class BIP66Test(eBitcashTestFramework):
    def __init__(self):
        super().__init__()
        self.num_nodes = 3
        self.setup_clean_chain = False
        self.extra_args = [[], ["-blockversion=2"], ["-blockversion=3"]]

    def setup_network(self):
        self.setup_nodes()
        connect_nodes(self.nodes[1], 0)
        connect_nodes(self.nodes[2], 0)
        self.sync_all()

    def run_test(self):
        cnt = self.nodes[0].getblockcount()

        # Mine some old-version blocks
        self.nodes[1].generate(100)
        self.sync_all()
        if (self.nodes[0].getblockcount() != cnt + 100):
            raise AssertionError("Failed to mine 100 version=2 blocks")

        # Mine 750 new-version blocks
        for i in range(15):
            self.nodes[2].generate(50)
        self.sync_all()
        if (self.nodes[0].getblockcount() != cnt + 850):
            raise AssertionError("Failed to mine 750 version=3 blocks")

        # TODO: check that new DERSIG rules are not enforced

        # Mine 1 new-version block
        self.nodes[2].generate(1)
        self.sync_all()
        if (self.nodes[0].getblockcount() != cnt + 851):
            raise AssertionError("Failed to mine a version=3 blocks")

        # TODO: check that new DERSIG rules are enforced

        # Mine 198 new-version blocks
        for i in range(2):
            self.nodes[2].generate(99)
        self.sync_all()
        if (self.nodes[0].getblockcount() != cnt + 1049):
            raise AssertionError("Failed to mine 198 version=3 blocks")

        # Mine 1 old-version block
        self.nodes[1].generate(1)
        self.sync_all()
        if (self.nodes[0].getblockcount() != cnt + 1050):
            raise AssertionError("Failed to mine a version=2 block after 949 version=3 blocks")

        # Mine 1 new-version blocks
        self.nodes[2].generate(1)
        self.sync_all()
        if (self.nodes[0].getblockcount() != cnt + 1051):
            raise AssertionError("Failed to mine a version=3 block")

        # Mine 1 old-version blocks. This should fail
        assert_raises_jsonrpc(-1, "CreateNewBlock: TestBlockValidity failed: bad-version(0x00000002)", self.nodes[1].generate, 1)
        self.sync_all()
        if (self.nodes[0].getblockcount() != cnt + 1051):
            raise AssertionError("Accepted a version=2 block after 950 version=3 blocks")

        # Mine 1 new-version blocks
        self.nodes[2].generate(1)
        self.sync_all()
        if (self.nodes[0].getblockcount() != cnt + 1052):
            raise AssertionError("Failed to mine a version=3 block")

if __name__ == '__main__':
    BIP66Test().main()


import sqlite3
import json
import math
from sqlite3.dbapi2 import Error

from flask import Flask, request, Response, render_template

app = Flask(__name__)

def open_db():
    db = sqlite3.connect('./transactions.db')
    db.row_factory = sqlite3.Row
    return db

@app.route('/', methods=['GET'])
def transactions():
    return render_template('transactions.html')

@app.route('/categories', methods=['GET'])
def categories():
    return render_template('categories.html')

@app.route('/api/transactions', methods=['GET'])
def get_transactions():
    with open_db() as db:
        results = db.execute('SELECT * FROM transactions WHERE date >= "2021-05-01" ORDER BY date ASC')
        return Response(json.dumps([dict(idx) for idx in results.fetchall()]), mimetype='application/json')

@app.route('/api/transactions/<int:id>', methods=['PUT', 'PATCH'])
def update_transaction(id):
    transaction = request.get_json(force=True)
    with open_db() as db:
        db.execute('UPDATE transactions SET category_id = ? WHERE id = ?', (transaction['category_id'], id))
        db.commit()
        return {'success': True}

@app.route('/api/categories', methods=['GET'])
def get_categories():
    with open_db() as db:
        results = db.execute('SELECT * FROM categories')
        return Response(json.dumps([dict(idx) for idx in results.fetchall()]), mimetype='application/json')

@app.route('/api/categories', methods=['POST'])
def create_category():
    category = request.get_json(force=True)
    with open_db() as db:
        db.execute('INSERT INTO categories (name) VALUES (?)', (category.get('name'),))
        db.commit()
        return {'success': True}

@app.route('/api/breakdown', methods=['GET'])
def get_breakdown():
    group_by_first = request.args.get('group_by', 'month').lower()

    with open_db() as db:
        results = db.execute('''
            SELECT
                j.*
            FROM (
                SELECT
                    t.id,
                    t.date,
                    SUBSTR(t.date, 0, 8) as month,
                    t.amount,
                    REPLACE(t.description, '  ', ' ') as description,
                    t.category_id,
                    c.name as category_name,
                    t.source
                FROM transactions t
                INNER JOIN categories c on t.category_id = c.id
                WHERE c.name NOT IN ('Income', 'Payments', 'Savings') AND t.date >= '2021-05'
            ) j
            ORDER BY j.month ASC, j.category_name ASC
        ''')
        # return Response(json.dumps([dict(idx) for idx in results.fetchall()], indent=2), mimetype='application/json')
        transactions = [dict(idx) for idx in results.fetchall()]

        if group_by_first == 'month':
            first_group = 'month'
            second_group = 'category_name'
        elif group_by_first == 'category':
            first_group = 'category_name'
            second_group = 'month'
        else:
            return Response(Error('Invalid group by'))

        aggregated_transactions = {}

        for item in transactions:
            item['description'] = item['description'].replace('  ', ' ', 10).replace('\t', ' ')

            top_group_value = item.get(first_group)
            second_group_value = item.get(second_group)
            if top_group_value in aggregated_transactions.keys():
                if second_group_value in aggregated_transactions[top_group_value].keys():
                    sub_group = aggregated_transactions[top_group_value][second_group_value]
                    sub_group['transactions'].append(item)
                    sub_group['summary']['amount'] += item['amount']
                    sub_group['summary']['total_transactions'] += 1
                    sub_group['summary']['min'] = min(sub_group['summary']['min'], item['amount'])
                    sub_group['summary']['max'] = max(sub_group['summary']['max'], item['amount'])
                    sub_group['summary']['avg'] = round(sub_group['summary']['amount'] / sub_group['summary']['total_transactions'], 2)
                else:
                    aggregated_transactions[top_group_value][second_group_value] = {
                        'summary': {
                            'amount': item['amount'],
                            'total_transactions': 1,
                            'min': item['amount'],
                            'max': item['amount'],
                            'avg': item['amount']
                        },
                        'transactions': [item]
                    }
            else:
                aggregated_transactions[top_group_value] = {}
                aggregated_transactions[top_group_value][second_group_value] = {
                    'summary': {
                        'amount': item['amount'],
                        'total_transactions': 1,
                        'min': item['amount'],
                        'max': item['amount'],
                        'avg': item['amount']
                    },
                    'transactions': [item]
                }

        return Response(json.dumps(aggregated_transactions, indent=2), mimetype='application/json')


load("@bazel_gazelle//:deps.bzl", "go_repository")

def go_repositories():
    go_repository(
        name = "co_honnef_go_tools",
        importpath = "honnef.co/go/tools",
        sum = "h1:/hemPrYIhOhy8zYrNj+069zDB68us2sMGsfkFJO0iZs=",
        version = "v0.0.0-20190523083050-ea95bdfd59fc",
    )
    go_repository(
        name = "com_github_burntsushi_toml",
        importpath = "github.com/BurntSushi/toml",
        sum = "h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=",
        version = "v0.3.1",
    )
    go_repository(
        name = "com_github_census_instrumentation_opencensus_proto",
        importpath = "github.com/census-instrumentation/opencensus-proto",
        sum = "h1:glEXhBS5PSLLv4IXzLA5yPRVX4bilULVyxxbrfOtDAk=",
        version = "v0.2.1",
    )
    go_repository(
        name = "com_github_client9_misspell",
        importpath = "github.com/client9/misspell",
        sum = "h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI=",
        version = "v0.3.4",
    )
    go_repository(
        name = "com_github_cncf_udpa_go",
        importpath = "github.com/cncf/udpa/go",
        sum = "h1:WBZRG4aNOuI15bLRrCgN8fCq8E5Xuty6jGbmSNEvSsU=",
        version = "v0.0.0-20191209042840-269d4d468f6f",
    )
    go_repository(
        name = "com_github_davecgh_go_spew",
        importpath = "github.com/davecgh/go-spew",
        sum = "h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=",
        version = "v1.1.0",
    )
    go_repository(
        name = "com_github_envoyproxy_go_control_plane",
        importpath = "github.com/envoyproxy/go-control-plane",
        sum = "h1:rEvIZUSZ3fx39WIi3JkQqQBitGwpELBIYWeBVh6wn+E=",
        version = "v0.9.4",
    )
    go_repository(
        name = "com_github_envoyproxy_protoc_gen_validate",
        importpath = "github.com/envoyproxy/protoc-gen-validate",
        sum = "h1:EQciDnbrYxy13PgWoY8AqoxGiPrpgBZ1R8UNe3ddc+A=",
        version = "v0.1.0",
    )
    go_repository(
        name = "com_github_golang_glog",
        importpath = "github.com/golang/glog",
        sum = "h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=",
        version = "v0.0.0-20160126235308-23def4e6c14b",
    )
    go_repository(
        name = "com_github_golang_mock",
        importpath = "github.com/golang/mock",
        sum = "h1:G5FRp8JnTd7RQH5kemVNlMeyXQAztQ3mOWV95KxsXH8=",
        version = "v1.1.1",
    )
    go_repository(
        name = "com_github_golang_protobuf",
        importpath = "github.com/golang/protobuf",
        sum = "h1:F768QJ1E9tib+q5Sc8MkdJi1RxLTbRcTf8LJV56aRls=",
        version = "v1.3.5",
    )
    go_repository(
        name = "com_github_google_go_cmp",
        importpath = "github.com/google/go-cmp",
        sum = "h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=",
        version = "v0.2.0",
    )
    go_repository(
        name = "com_github_labstack_echo",
        importpath = "github.com/labstack/echo",
        sum = "h1:pGRcYk231ExFAyoAjAfD85kQzRJCRI8bbnE7CX5OEgg=",
        version = "v3.3.10+incompatible",
    )
    go_repository(
        name = "com_github_labstack_gommon",
        importpath = "github.com/labstack/gommon",
        sum = "h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0=",
        version = "v0.3.0",
    )
    go_repository(
        name = "com_github_mattn_go_colorable",
        importpath = "github.com/mattn/go-colorable",
        sum = "h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=",
        version = "v0.1.2",
    )
    go_repository(
        name = "com_github_mattn_go_isatty",
        importpath = "github.com/mattn/go-isatty",
        sum = "h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg=",
        version = "v0.0.9",
    )
    go_repository(
        name = "com_github_pmezard_go_difflib",
        importpath = "github.com/pmezard/go-difflib",
        sum = "h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=",
        version = "v1.0.0",
    )
    go_repository(
        name = "com_github_prometheus_client_model",
        importpath = "github.com/prometheus/client_model",
        sum = "h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM=",
        version = "v0.0.0-20190812154241-14fe0d1b01d4",
    )
    go_repository(
        name = "com_github_stretchr_objx",
        importpath = "github.com/stretchr/objx",
        sum = "h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=",
        version = "v0.1.0",
    )
    go_repository(
        name = "com_github_stretchr_testify",
        importpath = "github.com/stretchr/testify",
        sum = "h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=",
        version = "v1.5.1",
    )
    go_repository(
        name = "com_github_valyala_bytebufferpool",
        importpath = "github.com/valyala/bytebufferpool",
        sum = "h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=",
        version = "v1.0.0",
    )
    go_repository(
        name = "com_github_valyala_fasttemplate",
        importpath = "github.com/valyala/fasttemplate",
        sum = "h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8=",
        version = "v1.0.1",
    )
    go_repository(
        name = "com_google_cloud_go",
        importpath = "cloud.google.com/go",
        sum = "h1:e0WKqKTd5BnrG8aKH3J3h+QvEIQtSUcf2n5UZ5ZgLtQ=",
        version = "v0.26.0",
    )
    go_repository(
        name = "in_gopkg_check_v1",
        importpath = "gopkg.in/check.v1",
        sum = "h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=",
        version = "v0.0.0-20161208181325-20d25e280405",
    )
    go_repository(
        name = "in_gopkg_yaml_v2",
        importpath = "gopkg.in/yaml.v2",
        sum = "h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=",
        version = "v2.2.2",
    )
    go_repository(
        name = "org_golang_google_appengine",
        importpath = "google.golang.org/appengine",
        sum = "h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=",
        version = "v1.4.0",
    )
    go_repository(
        name = "org_golang_google_genproto",
        importpath = "google.golang.org/genproto",
        sum = "h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE=",
        version = "v0.0.0-20190819201941-24fa4b261c55",
    )
    go_repository(
        name = "org_golang_google_grpc",
        importpath = "google.golang.org/grpc",
        sum = "h1:bO/TA4OxCOummhSf10siHuG7vJOiwh7SpRpFZDkOgl4=",
        version = "v1.28.0",
    )
    go_repository(
        name = "org_golang_x_crypto",
        importpath = "golang.org/x/crypto",
        sum = "h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=",
        version = "v0.0.0-20190308221718-c2843e01d9a2",
    )
    go_repository(
        name = "org_golang_x_exp",
        importpath = "golang.org/x/exp",
        sum = "h1:c2HOrn5iMezYjSlGPncknSEr/8x5LELb/ilJbXi9DEA=",
        version = "v0.0.0-20190121172915-509febef88a4",
    )
    go_repository(
        name = "org_golang_x_lint",
        importpath = "golang.org/x/lint",
        sum = "h1:XQyxROzUlZH+WIQwySDgnISgOivlhjIEwaQaJEJrrN0=",
        version = "v0.0.0-20190313153728-d0100b6bd8b3",
    )
    go_repository(
        name = "org_golang_x_net",
        importpath = "golang.org/x/net",
        sum = "h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628=",
        version = "v0.0.0-20190311183353-d8887717615a",
    )
    go_repository(
        name = "org_golang_x_oauth2",
        importpath = "golang.org/x/oauth2",
        sum = "h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs=",
        version = "v0.0.0-20180821212333-d2e6202438be",
    )
    go_repository(
        name = "org_golang_x_sync",
        importpath = "golang.org/x/sync",
        sum = "h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=",
        version = "v0.0.0-20190423024810-112230192c58",
    )
    go_repository(
        name = "org_golang_x_sys",
        importpath = "golang.org/x/sys",
        sum = "h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ=",
        version = "v0.0.0-20190813064441-fde4db37ae7a",
    )
    go_repository(
        name = "org_golang_x_text",
        importpath = "golang.org/x/text",
        sum = "h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=",
        version = "v0.3.0",
    )
    go_repository(
        name = "org_golang_x_tools",
        importpath = "golang.org/x/tools",
        sum = "h1:5Beo0mZN8dRzgrMMkDp0jc8YXQKx9DiJ2k1dkvGsn5A=",
        version = "v0.0.0-20190524140312-2c0ae7006135",
    )


"""A collection of tasks."""
import logging

from ..const import AddonState
from ..coresys import CoreSysAttributes
from ..exceptions import (
    AddonsError,
    AudioError,
    CliError,
    CoreDNSError,
    HomeAssistantError,
    MulticastError,
    ObserverError,
)
from ..host.const import HostFeature
from ..jobs.decorator import Job, JobCondition

_LOGGER: logging.Logger = logging.getLogger(__name__)

HASS_WATCHDOG_API = "HASS_WATCHDOG_API"

RUN_UPDATE_SUPERVISOR = 29100
RUN_UPDATE_ADDONS = 57600
RUN_UPDATE_CLI = 28100
RUN_UPDATE_DNS = 30100
RUN_UPDATE_AUDIO = 30200
RUN_UPDATE_MULTICAST = 30300
RUN_UPDATE_OBSERVER = 30400

RUN_RELOAD_ADDONS = 10800
RUN_RELOAD_BACKUPS = 72000
RUN_RELOAD_HOST = 7600
RUN_RELOAD_UPDATER = 7200
RUN_RELOAD_INGRESS = 930

RUN_WATCHDOG_HOMEASSISTANT_DOCKER = 15
RUN_WATCHDOG_HOMEASSISTANT_API = 120

RUN_WATCHDOG_DNS_DOCKER = 30
RUN_WATCHDOG_AUDIO_DOCKER = 60
RUN_WATCHDOG_CLI_DOCKER = 60
RUN_WATCHDOG_OBSERVER_DOCKER = 60
RUN_WATCHDOG_MULTICAST_DOCKER = 60

RUN_WATCHDOG_ADDON_DOCKER = 30
RUN_WATCHDOG_ADDON_APPLICATON = 120
RUN_WATCHDOG_OBSERVER_APPLICATION = 180

RUN_REFRESH_ADDON = 15

RUN_CHECK_CONNECTIVITY = 30

class Tasks(CoreSysAttributes):
    """Handle Tasks inside Supervisor."""

    def __init__(self, coresys):
        """Initialize Tasks."""
        self.coresys = coresys
        self._cache = {}

    async def load(self):
        """Add Tasks to scheduler."""
        # Update
        self.sys_scheduler.register_task(self._update_addons, RUN_UPDATE_ADDONS)
        self.sys_scheduler.register_task(self._update_supervisor, RUN_UPDATE_SUPERVISOR)
        self.sys_scheduler.register_task(self._update_cli, RUN_UPDATE_CLI)
        self.sys_scheduler.register_task(self._update_dns, RUN_UPDATE_DNS)
        self.sys_scheduler.register_task(self._update_audio, RUN_UPDATE_AUDIO)
        self.sys_scheduler.register_task(self._update_multicast, RUN_UPDATE_MULTICAST)
        self.sys_scheduler.register_task(self._update_observer, RUN_UPDATE_OBSERVER)

        # Reload
        self.sys_scheduler.register_task(self.sys_store.reload, RUN_RELOAD_ADDONS)
        self.sys_scheduler.register_task(self.sys_updater.reload, RUN_RELOAD_UPDATER)
        self.sys_scheduler.register_task(self.sys_backups.reload, RUN_RELOAD_BACKUPS)
        self.sys_scheduler.register_task(self.sys_host.reload, RUN_RELOAD_HOST)
        self.sys_scheduler.register_task(self.sys_ingress.reload, RUN_RELOAD_INGRESS)

        # Watchdog
        self.sys_scheduler.register_task(
            self._watchdog_homeassistant_docker, RUN_WATCHDOG_HOMEASSISTANT_DOCKER
        )
        self.sys_scheduler.register_task(
            self._watchdog_homeassistant_api, RUN_WATCHDOG_HOMEASSISTANT_API
        )
        self.sys_scheduler.register_task(
            self._watchdog_dns_docker, RUN_WATCHDOG_DNS_DOCKER
        )
        self.sys_scheduler.register_task(
            self._watchdog_audio_docker, RUN_WATCHDOG_AUDIO_DOCKER
        )
        self.sys_scheduler.register_task(
            self._watchdog_cli_docker, RUN_WATCHDOG_CLI_DOCKER
        )
        self.sys_scheduler.register_task(
            self._watchdog_observer_docker, RUN_WATCHDOG_OBSERVER_DOCKER
        )
        self.sys_scheduler.register_task(
            self._watchdog_observer_application, RUN_WATCHDOG_OBSERVER_APPLICATION
        )
        self.sys_scheduler.register_task(
            self._watchdog_multicast_docker, RUN_WATCHDOG_MULTICAST_DOCKER
        )
        self.sys_scheduler.register_task(
            self._watchdog_addon_docker, RUN_WATCHDOG_ADDON_DOCKER
        )
        self.sys_scheduler.register_task(
            self._watchdog_addon_application, RUN_WATCHDOG_ADDON_APPLICATON
        )

        # Refresh
        self.sys_scheduler.register_task(self._refresh_addon, RUN_REFRESH_ADDON)

        # Connectivity
        self.sys_scheduler.register_task(
            self._check_connectivity, RUN_CHECK_CONNECTIVITY
        )

        _LOGGER.info("All core tasks are scheduled")

    @Job(
        conditions=[
            JobCondition.HEALTHY,
            JobCondition.FREE_SPACE,
            JobCondition.INTERNET_HOST,
            JobCondition.RUNNING,
        ]
    )
    async def _update_addons(self):
        """Check if an update is available for an Add-on and update it."""
        for addon in self.sys_addons.all:
            if not addon.is_installed or not addon.auto_update:
                continue

            # Evaluate available updates
            if not addon.need_update:
                continue
            if not addon.test_update_schema():
                _LOGGER.warning(
                    "Add-on %s will be ignored, schema tests failed", addon.slug
                )
                continue

            # Run Add-on update sequential
            # avoid issue on slow IO
            _LOGGER.info("Add-on auto update process %s", addon.slug)
            try:
                await addon.update(backup=True)
            except AddonsError:
                _LOGGER.error("Can't auto update Add-on %s", addon.slug)

    @Job(
        conditions=[
            JobCondition.FREE_SPACE,
            JobCondition.INTERNET_HOST,
            JobCondition.RUNNING,
        ]
    )
    async def _update_supervisor(self):
        """Check and run update of Supervisor Supervisor."""
        if not self.sys_supervisor.need_update:
            return

        _LOGGER.info(
            "Found new Supervisor version %s, updating",
            self.sys_supervisor.latest_version,
        )
        await self.sys_supervisor.update()

    async def _watchdog_homeassistant_docker(self):
        """Check running state of Docker and start if they is close."""
        if not self.sys_homeassistant.watchdog:
            # Watchdog is not enabled for Home Assistant
            return
        if self.sys_homeassistant.error_state:
            # Home Assistant is in an error state, this is handled by the rollback feature
            return
        if not await self.sys_homeassistant.core.is_failed():
            # The home assistant container is not in a failed state
            return
        if self.sys_homeassistant.core.in_progress:
            # Home Assistant has a task in progress
            return
        if await self.sys_homeassistant.core.is_running():
            # Home Assistant is running
            return

        _LOGGER.warning("Watchdog found a problem with Home Assistant Docker!")
        try:
            await self.sys_homeassistant.core.start()
        except HomeAssistantError as err:
            _LOGGER.error("Home Assistant watchdog reanimation failed!")
            self.sys_capture_exception(err)
        else:
            return

        _LOGGER.info("Rebuilding the Home Assistant Container")
        await self.sys_homeassistant.core.rebuild()

    async def _watchdog_homeassistant_api(self):
        """Create scheduler task for monitoring running state of API.

        Try 2 times to call API before we restart Home-Assistant. Maybe we had
        a delay in our system.
        """
        if not self.sys_homeassistant.watchdog:
            # Watchdog is not enabled for Home Assistant
            return
        if self.sys_homeassistant.error_state:
            # Home Assistant is in an error state, this is handled by the rollback feature
            return
        if not await self.sys_homeassistant.core.is_running():
            # The home assistant container is not running
            return
        if self.sys_homeassistant.core.in_progress:
            # Home Assistant has a task in progress
            return
        if await self.sys_homeassistant.api.check_api_state():
            # Home Assistant is running properly
            return

        # Init cache data
        retry_scan = self._cache.get(HASS_WATCHDOG_API, 0)

        # Look like we run into a problem
        retry_scan += 1
        if retry_scan == 1:
            self._cache[HASS_WATCHDOG_API] = retry_scan
            _LOGGER.warning("Watchdog miss API response from Home Assistant")
            return

        _LOGGER.error("Watchdog found a problem with Home Assistant API!")
        try:
            await self.sys_homeassistant.core.restart()
        except HomeAssistantError as err:
            _LOGGER.error("Home Assistant watchdog reanimation failed!")
            self.sys_capture_exception(err)
        finally:
            self._cache[HASS_WATCHDOG_API] = 0

    @Job(conditions=JobCondition.RUNNING)
    async def _update_cli(self):
        """Check and run update of cli."""
        if not self.sys_plugins.cli.need_update:
            return

        _LOGGER.info(
            "Found new cli version %s, updating", self.sys_plugins.cli.latest_version
        )
        await self.sys_plugins.cli.update()

    @Job(conditions=JobCondition.RUNNING)
    async def _update_dns(self):
        """Check and run update of CoreDNS plugin."""
        if not self.sys_plugins.dns.need_update:
            return

        _LOGGER.info(
            "Found new CoreDNS plugin version %s, updating",
            self.sys_plugins.dns.latest_version,
        )
        await self.sys_plugins.dns.update()

    @Job(conditions=JobCondition.RUNNING)
    async def _update_audio(self):
        """Check and run update of PulseAudio plugin."""
        if not self.sys_plugins.audio.need_update:
            return

        _LOGGER.info(
            "Found new PulseAudio plugin version %s, updating",
            self.sys_plugins.audio.latest_version,
        )
        await self.sys_plugins.audio.update()

    @Job(conditions=JobCondition.RUNNING)
    async def _update_observer(self):
        """Check and run update of Observer plugin."""
        if not self.sys_plugins.observer.need_update:
            return

        _LOGGER.info(
            "Found new Observer plugin version %s, updating",
            self.sys_plugins.observer.latest_version,
        )
        await self.sys_plugins.observer.update()

    @Job(conditions=JobCondition.RUNNING)
    async def _update_multicast(self):
        """Check and run update of multicast."""
        if not self.sys_plugins.multicast.need_update:
            return

        _LOGGER.info(
            "Found new Multicast version %s, updating",
            self.sys_plugins.multicast.latest_version,
        )
        await self.sys_plugins.multicast.update()

    async def _watchdog_dns_docker(self):
        """Check running state of Docker and start if they is close."""
        # if CoreDNS is active
        if await self.sys_plugins.dns.is_running() or self.sys_plugins.dns.in_progress:
            return
        _LOGGER.warning("Watchdog found a problem with CoreDNS plugin!")

        # Detect loop
        await self.sys_plugins.dns.loop_detection()

        try:
            await self.sys_plugins.dns.start()
        except CoreDNSError:
            _LOGGER.error("CoreDNS watchdog reanimation failed!")

    async def _watchdog_audio_docker(self):
        """Check running state of Docker and start if they is close."""
        # if PulseAudio plugin is active
        if (
            await self.sys_plugins.audio.is_running()
            or self.sys_plugins.audio.in_progress
        ):
            return
        _LOGGER.warning("Watchdog found a problem with PulseAudio plugin!")

        try:
            await self.sys_plugins.audio.start()
        except AudioError:
            _LOGGER.error("PulseAudio watchdog reanimation failed!")

    async def _watchdog_cli_docker(self):
        """Check running state of Docker and start if they is close."""
        # if cli plugin is active
        if await self.sys_plugins.cli.is_running() or self.sys_plugins.cli.in_progress:
            return
        _LOGGER.warning("Watchdog found a problem with cli plugin!")

        try:
            await self.sys_plugins.cli.start()
        except CliError:
            _LOGGER.error("CLI watchdog reanimation failed!")

    async def _watchdog_observer_docker(self):
        """Check running state of Docker and start if they is close."""
        # if observer plugin is active
        if (
            await self.sys_plugins.observer.is_running()
            or self.sys_plugins.observer.in_progress
        ):
            return
        _LOGGER.warning("Watchdog/Docker found a problem with observer plugin!")

        try:
            await self.sys_plugins.observer.start()
        except ObserverError:
            _LOGGER.error("Observer watchdog reanimation failed!")

    async def _watchdog_observer_application(self):
        """Check running state of application and rebuild if they is not response."""
        # if observer plugin is active
        if (
            self.sys_plugins.observer.in_progress
            or await self.sys_plugins.observer.check_system_runtime()
        ):
            return
        _LOGGER.warning("Watchdog/Application found a problem with observer plugin!")

        try:
            await self.sys_plugins.observer.rebuild()
        except ObserverError:
            _LOGGER.error("Observer watchdog reanimation failed!")

    async def _watchdog_multicast_docker(self):
        """Check running state of Docker and start if they is close."""
        # if multicast plugin is active
        if (
            await self.sys_plugins.multicast.is_running()
            or self.sys_plugins.multicast.in_progress
        ):
            return
        _LOGGER.warning("Watchdog found a problem with Multicast plugin!")

        try:
            await self.sys_plugins.multicast.start()
        except MulticastError:
            _LOGGER.error("Multicast watchdog reanimation failed!")

    async def _watchdog_addon_docker(self):
        """Check running state  of Docker and start if they is close."""
        for addon in self.sys_addons.installed:
            # if watchdog need looking for
            if not addon.watchdog or await addon.is_running():
                continue

            # if Addon have running actions
            if addon.in_progress or addon.state != AddonState.STARTED:
                continue

            _LOGGER.warning("Watchdog found a problem with %s!", addon.slug)
            try:
                await addon.start()
            except AddonsError as err:
                _LOGGER.error("%s watchdog reanimation failed with %s", addon.slug, err)
                self.sys_capture_exception(err)

    async def _watchdog_addon_application(self):
        """Check running state of the application and start if they is hangs."""
        for addon in self.sys_addons.installed:
            # if watchdog need looking for
            if not addon.watchdog or addon.state != AddonState.STARTED:
                continue

            # Init cache data
            retry_scan = self._cache.get(addon.slug, 0)

            # if Addon have running actions / Application work
            if addon.in_progress or await addon.watchdog_application():
                continue

            # Look like we run into a problem
            retry_scan += 1
            if retry_scan == 1:
                self._cache[addon.slug] = retry_scan
                _LOGGER.warning(
                    "Watchdog missing application response from %s", addon.slug
                )
                return

            _LOGGER.warning("Watchdog found a problem with %s application!", addon.slug)
            try:
                await addon.restart()
            except AddonsError as err:
                _LOGGER.error("%s watchdog reanimation failed with %s", addon.slug, err)
                self.sys_capture_exception(err)
            finally:
                self._cache[addon.slug] = 0

    async def _refresh_addon(self) -> None:
        """Refresh addon state."""
        for addon in self.sys_addons.installed:
            # if watchdog need looking for
            if addon.watchdog or addon.state != AddonState.STARTED:
                continue

            # if Addon have running actions
            if addon.in_progress or await addon.is_running():
                continue

            # Adjust state
            addon.state = AddonState.STOPPED

    async def _check_connectivity(self) -> None:
        """Check system connectivity."""
        value = self._cache.get("connectivity", 0)

        # Need only full check if not connected or each 10min
        if value >= 600:
            pass
        elif (
            self.sys_supervisor.connectivity
            and self.sys_host.network.connectivity is None
        ) or (
            self.sys_supervisor.connectivity
            and self.sys_host.network.connectivity is not None
            and self.sys_host.network.connectivity
        ):
            self._cache["connectivity"] = value + RUN_CHECK_CONNECTIVITY
            return

        # Check connectivity
        try:
            await self.sys_supervisor.check_connectivity()
            if HostFeature.NETWORK in self.sys_host.features:
                await self.sys_host.network.check_connectivity()
        finally:
            self._cache["connectivity"] = 0


"""Patch to fix MNIST download issue as described here:
- https://github.com/pytorch/ignite/issues/1737
- https://github.com/pytorch/vision/issues/3500
"""

import os
import subprocess as sp

import torch
from torchvision.datasets.mnist import MNIST, read_image_file, read_label_file
from torchvision.datasets.utils import extract_archive

def patched_download(self):
    """wget patched download method.
    """
    if self._check_exists():
        return

    os.makedirs(self.raw_folder, exist_ok=True)
    os.makedirs(self.processed_folder, exist_ok=True)

    # download files
    for url, md5 in self.resources:
        filename = url.rpartition("/")[2]
        download_root = os.path.expanduser(self.raw_folder)
        extract_root = None
        remove_finished = False

        if extract_root is None:
            extract_root = download_root
        if not filename:
            filename = os.path.basename(url)

        # Use wget to download archives
        sp.run(["wget", url, "-P", download_root])

        archive = os.path.join(download_root, filename)
        print("Extracting {} to {}".format(archive, extract_root))
        extract_archive(archive, extract_root, remove_finished)

    # process and save as torch files
    print("Processing...")

    training_set = (
        read_image_file(os.path.join(self.raw_folder, "train-images-idx3-ubyte")),
        read_label_file(os.path.join(self.raw_folder, "train-labels-idx1-ubyte")),
    )
    test_set = (
        read_image_file(os.path.join(self.raw_folder, "t10k-images-idx3-ubyte")),
        read_label_file(os.path.join(self.raw_folder, "t10k-labels-idx1-ubyte")),
    )
    with open(os.path.join(self.processed_folder, self.training_file), "wb") as f:
        torch.save(training_set, f)
    with open(os.path.join(self.processed_folder, self.test_file), "wb") as f:
        torch.save(test_set, f)

    print("Done!")

def main():
    # Patch download method
    MNIST.download = patched_download
    # Download MNIST
    MNIST(".", download=True)

if __name__ == "__main__":
    main()


from datetime import datetime
from os.path import dirname, join

import pytest  # noqa
from city_scrapers_core.constants import COMMITTEE, PASSED
from city_scrapers_core.utils import file_response
from freezegun import freeze_time

from city_scrapers.spiders.cuya_emergency_services_advisory import (
    CuyaEmergencyServicesAdvisorySpider,
)

test_response = file_response(
    join(dirname(__file__), "files", "cuya_emergency_services_advisory.html"),
    url="http://bc.cuyahogacounty.us/en-US/CC-EmergencySrvcsAdvsryBrd.aspx",
)
test_detail_response = file_response(
    join(dirname(__file__), "files", "cuya_emergency_services_advisory_detail.html"),
    url="http://bc.cuyahogacounty.us/en-US/091119-CCESAB-Comms-meeting.aspx",
)
spider = CuyaEmergencyServicesAdvisorySpider()

freezer = freeze_time("2019-09-25")
freezer.start()

parsed_items = [item for item in spider.parse(test_response)]
parsed_item = [item for item in spider._parse_detail(test_detail_response)][0]

freezer.stop()

def test_count():
    assert len(parsed_items) == 30

def test_title():
    assert parsed_item["title"] == "CCESAB Communications Committee"

def test_description():
    assert parsed_item["description"] == ""

def test_start():
    assert parsed_item["start"] == datetime(2019, 9, 11, 10, 15)

def test_end():
    assert parsed_item["end"] == datetime(2019, 9, 11, 11, 30)

def test_time_notes():
    assert parsed_item["time_notes"] == ""

def test_id():
    assert (
        parsed_item["id"]
        == "cuya_emergency_services_advisory/201909111015/x/ccesab_communications_committee"  # noqa
    )

def test_status():
    assert parsed_item["status"] == PASSED

def test_location():
    assert parsed_item["location"] == {
        "name": "The Cassidy Theatre",
        "address": "6200 Pearl Road Parma Heights, OH 44130",
    }

def test_source():
    assert parsed_item["source"] == test_detail_response.url

def test_links():
    assert parsed_item["links"] == [
        {
            "href": "http://bc.cuyahogacounty.us/ViewFile.aspx?file=7DSCAKoM0rqkeTzD%2f6%2f4cw%3d%3d",  # noqa
            "title": "Agenda",
        }
    ]

def test_classification():
    assert parsed_item["classification"] == COMMITTEE

def test_all_day():
    assert parsed_item["all_day"] is False


# -*- coding: utf-8 -*-
# :Project:   metapensiero.pj -- compatibility
# :Created:   lun 30 mar 2020, 01:48:33
# :Author:    Alberto Berti <alberto@metapensiero.it>
# :License:   GNU General Public License version 3 or later
# :Copyright: © 2020 Alberto Berti
#

import ast
import sys

is_py36 = sys.version_info >= (3, 6)

if is_py36:
    assign_types = (ast.Assign, ast.AnnAssign)
else:
    assign_types = (ast.Assign,)


import math

from os.path import join as pjoin
from collections import OrderedDict

import torch
import torch.nn as nn
import torch.nn.functional as F

class DoubleConv(nn.Module):
    def __init__(self, in_channels, out_channels):
        super(DoubleConv, self).__init__()
        self.conv = nn.Sequential(
            nn.Conv2d(in_channels, out_channels, 3, 1, 1, bias=False),
            nn.BatchNorm2d(out_channels),
            nn.ReLU(inplace=True),
            nn.Conv2d(out_channels, out_channels, 3, 1, 1, bias=False),
            nn.BatchNorm2d(out_channels),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(kernel_size=2, stride=2)
        )
    def forward(self, x):
        return self.conv(x)

class SingleConv_no_pool(nn.Module):
    def __init__(self, in_channels, out_channels):
        super(SingleConv_no_pool, self).__init__()
        self.conv = nn.Sequential(
            nn.Conv2d(in_channels, out_channels, 3, 1, 1, bias=False),
            nn.BatchNorm2d(out_channels),
            nn.ReLU(inplace=True),
            nn.Conv2d(out_channels, out_channels, 3, 1, 1, bias=False),
            nn.BatchNorm2d(out_channels),
            nn.ReLU(inplace=True),
        )
    def forward(self, x):
        if x.size()[1]== 1:
            x = x.repeat(1,3,1,1)
        return self.conv(x)

class SingleConv_with_pool(nn.Module):
    def __init__(self, in_channels, out_channels):
        super(SingleConv_with_pool, self).__init__()
        self.conv = nn.Sequential(
            nn.Conv2d(in_channels, out_channels, 3, 1, 1, bias=False),
            nn.BatchNorm2d(out_channels),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(kernel_size=2, stride=2)
        )
    def forward(self, x):
        return self.conv(x)

class UNET_encoder(nn.Module):

    def __init__(self):
        super().__init__()
        width = 32
        self.width = width

        self.root = nn.Sequential(OrderedDict([
            ('unit1', SingleConv_no_pool(3, width))
        ]))

        self.body = nn.Sequential(OrderedDict([
            ('block1', nn.Sequential(OrderedDict(
                [('unit2', SingleConv_with_pool(width, width*2))]
                ))),
            ('block2', nn.Sequential(OrderedDict(
                [('unit3', DoubleConv(width*2, width*4))]
                ))),
            ('block3', nn.Sequential(OrderedDict(
                [('unit4', DoubleConv(width*4, width*8))]
                ))),
            ('block4', nn.Sequential(OrderedDict(
                [('unit5', DoubleConv(width*8, width*16))]
                ))),
        ]))

    def forward(self, x):
        features = []
        x = self.root(x)
        b, c, in_size, _ = x.size()
        features.append(x)
        for i in range(len(self.body)-1):
            x = self.body[i](x)
            features.append(x)
        x = self.body[-1](x)
        return x, features[::-1]

class UNET_encoder_FETS(nn.Module):

    def __init__(self):
        super().__init__()
        width = 32
        self.width = width

        self.root = nn.Sequential(OrderedDict([
            ('unit1', SingleConv_no_pool(4, width))
        ]))

        self.body = nn.Sequential(OrderedDict([
            ('block1', nn.Sequential(OrderedDict(
                [('unit2', SingleConv_with_pool(width, width*2))]
                ))),
            ('block2', nn.Sequential(OrderedDict(
                [('unit3', DoubleConv(width*2, width*4))]
                ))),
            ('block3', nn.Sequential(OrderedDict(
                [('unit4', DoubleConv(width*4, width*8))]
                ))),
            ('block4', nn.Sequential(OrderedDict(
                [('unit5', DoubleConv(width*8, width*16))]
                ))),
        ]))

    def forward(self, x):
        features = []
        x = self.root(x)
        b, c, in_size, _ = x.size()
        features.append(x)
        for i in range(len(self.body)-1):
            x = self.body[i](x)
            features.append(x)
        x = self.body[-1](x)
        return x, features[::-1]


# ##### BEGIN GPL LICENSE BLOCK #####
#
#  This program is free software; you can redistribute it and/or
#  modify it under the terms of the GNU General Public License
#  as published by the Free Software Foundation; either version 2
#  of the License, or (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program; if not, write to the Free Software Foundation,
#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# ##### END GPL LICENSE BLOCK #####

# <pep8-80 compliant>

from _bpy import types as bpy_types
import _bpy

StructRNA = bpy_types.bpy_struct
StructMetaPropGroup = bpy_types.bpy_struct_meta_idprop
# StructRNA = bpy_types.Struct

bpy_types.BlendDataLibraries.load = _bpy._library_load
bpy_types.BlendDataLibraries.write = _bpy._library_write
bpy_types.BlendData.user_map = _bpy._rna_id_collection_user_map
bpy_types.BlendData.batch_remove = _bpy._rna_id_collection_batch_remove

class Context(StructRNA):
    __slots__ = ()

    def copy(self):
        from types import BuiltinMethodType
        new_context = {}
        generic_attrs = (
            *StructRNA.__dict__.keys(),
            "bl_rna", "rna_type", "copy",
        )
        for attr in dir(self):
            if not (attr.startswith("_") or attr in generic_attrs):
                value = getattr(self, attr)
                if type(value) != BuiltinMethodType:
                    new_context[attr] = value

        return new_context

class Library(bpy_types.ID):
    __slots__ = ()

    @property
    def users_id(self):
        """ID data blocks which use this library"""
        import bpy

        # See: readblenentry.c, IDTYPE_FLAGS_ISLINKABLE,
        # we could make this an attribute in rna.
        attr_links = (
            "actions", "armatures", "brushes", "cameras",
            "curves", "grease_pencils", "collections", "images",
            "lights", "lattices", "materials", "metaballs",
            "meshes", "node_groups", "objects", "scenes",
            "sounds", "speakers", "textures", "texts",
            "fonts", "worlds",
        )

        return tuple(id_block
                     for attr in attr_links
                     for id_block in getattr(bpy.data, attr)
                     if id_block.library == self)

class Texture(bpy_types.ID):
    __slots__ = ()

    @property
    def users_material(self):
        """Materials that use this texture"""
        import bpy
        return tuple(mat for mat in bpy.data.materials
                     if self in [slot.texture
                                 for slot in mat.texture_slots
                                 if slot]
                     )

    @property
    def users_object_modifier(self):
        """Object modifiers that use this texture"""
        import bpy
        return tuple(
            obj for obj in bpy.data.objects if
            self in [
                mod.texture
                for mod in obj.modifiers
                if mod.type == 'DISPLACE']
        )

class Collection(bpy_types.ID):
    __slots__ = ()

    @property
    def users_dupli_group(self):
        """The collection instance objects this collection is used in"""
        import bpy
        return tuple(obj for obj in bpy.data.objects
                     if self == obj.instance_collection)

class Object(bpy_types.ID):
    __slots__ = ()

    @property
    def children(self):
        """All the children of this object. Warning: takes O(len(bpy.data.objects)) time."""
        import bpy
        return tuple(child for child in bpy.data.objects
                     if child.parent == self)

    @property
    def users_collection(self):
        """The collections this object is in. Warning: takes O(len(bpy.data.collections) + len(bpy.data.scenes)) time."""
        import bpy
        return (
            tuple(
                collection for collection in bpy.data.collections
                if self in collection.objects[:]
            ) + tuple(
                scene.collection for scene in bpy.data.scenes
                if self in scene.collection.objects[:]
            )
        )

    @property
    def users_scene(self):
        """The scenes this object is in. Warning: takes O(len(bpy.data.scenes) * len(bpy.data.objects)) time."""
        import bpy
        return tuple(scene for scene in bpy.data.scenes
                     if self in scene.objects[:])

class WindowManager(bpy_types.ID):
    __slots__ = ()

    def popup_menu(self, draw_func, title="", icon='NONE'):
        import bpy
        popup = self.popmenu_begin__internal(title, icon=icon)

        try:
            draw_func(popup, bpy.context)
        finally:
            self.popmenu_end__internal(popup)

    def popover(
            self, draw_func, *,
            ui_units_x=0,
            keymap=None,
            from_active_button=False,
    ):
        import bpy
        popup = self.popover_begin__internal(
            ui_units_x=ui_units_x,
            from_active_button=from_active_button,
        )

        try:
            draw_func(popup, bpy.context)
        finally:
            self.popover_end__internal(popup, keymap=keymap)

    def popup_menu_pie(self, event, draw_func, title="", icon='NONE'):
        import bpy
        pie = self.piemenu_begin__internal(title, icon=icon, event=event)

        if pie:
            try:
                draw_func(pie, bpy.context)
            finally:
                self.piemenu_end__internal(pie)

class WorkSpace(bpy_types.ID):
    __slots__ = ()

    def status_text_set(self, text):
        """
        Set the status text or None to clear,
        When text is a function, this will be called with the (header, context) arguments.
        """
        from bl_ui.space_statusbar import STATUSBAR_HT_header
        draw_fn = getattr(STATUSBAR_HT_header, "_draw_orig", None)
        if draw_fn is None:
            draw_fn = STATUSBAR_HT_header._draw_orig = STATUSBAR_HT_header.draw

        if not (text is None or isinstance(text, str)):
            draw_fn = text
            text = None

        self.status_text_set_internal(text)
        STATUSBAR_HT_header.draw = draw_fn

class _GenericBone:
    """
    functions for bones, common between Armature/Pose/Edit bones.
    internal subclassing use only.
    """
    __slots__ = ()

    def translate(self, vec):
        """Utility function to add *vec* to the head and tail of this bone"""
        self.head += vec
        self.tail += vec

    def parent_index(self, parent_test):
        """
        The same as 'bone in other_bone.parent_recursive'
        but saved generating a list.
        """
        # use the name so different types can be tested.
        name = parent_test.name

        parent = self.parent
        i = 1
        while parent:
            if parent.name == name:
                return i
            parent = parent.parent
            i += 1

        return 0

    @property
    def x_axis(self):
        """ Vector pointing down the x-axis of the bone.
        """
        from mathutils import Vector
        return self.matrix.to_3x3() @ Vector((1.0, 0.0, 0.0))

    @property
    def y_axis(self):
        """ Vector pointing down the y-axis of the bone.
        """
        from mathutils import Vector
        return self.matrix.to_3x3() @ Vector((0.0, 1.0, 0.0))

    @property
    def z_axis(self):
        """ Vector pointing down the z-axis of the bone.
        """
        from mathutils import Vector
        return self.matrix.to_3x3() @ Vector((0.0, 0.0, 1.0))

    @property
    def basename(self):
        """The name of this bone before any '.' character"""
        # return self.name.rsplit(".", 1)[0]
        return self.name.split(".")[0]

    @property
    def parent_recursive(self):
        """A list of parents, starting with the immediate parent"""
        parent_list = []
        parent = self.parent

        while parent:
            if parent:
                parent_list.append(parent)

            parent = parent.parent

        return parent_list

    @property
    def center(self):
        """The midpoint between the head and the tail."""
        return (self.head + self.tail) * 0.5

    @property
    def vector(self):
        """
        The direction this bone is pointing.
        Utility function for (tail - head)
        """
        return (self.tail - self.head)

    @property
    def children(self):
        """A list of all the bones children. Warning: takes O(len(bones)) time."""
        return [child for child in self._other_bones if child.parent == self]

    @property
    def children_recursive(self):
        """A list of all children from this bone. Warning: takes O(len(bones)**2) time."""
        bones_children = []
        for bone in self._other_bones:
            index = bone.parent_index(self)
            if index:
                bones_children.append((index, bone))

        # sort by distance to parent
        bones_children.sort(key=lambda bone_pair: bone_pair[0])
        return [bone for index, bone in bones_children]

    @property
    def children_recursive_basename(self):
        """
        Returns a chain of children with the same base name as this bone.
        Only direct chains are supported, forks caused by multiple children
        with matching base names will terminate the function
        and not be returned. Warning: takes O(len(bones)**2) time.
        """
        basename = self.basename
        chain = []

        child = self
        while True:
            children = child.children
            children_basename = []

            for child in children:
                if basename == child.basename:
                    children_basename.append(child)

            if len(children_basename) == 1:
                child = children_basename[0]
                chain.append(child)
            else:
                if children_basename:
                    print("multiple basenames found, "
                          "this is probably not what you want!",
                          self.name, children_basename)

                break

        return chain

    @property
    def _other_bones(self):
        id_data = self.id_data
        id_data_type = type(id_data)

        if id_data_type == bpy_types.Object:
            bones = id_data.pose.bones
        elif id_data_type == bpy_types.Armature:
            bones = id_data.edit_bones
            if not bones:  # not in edit mode
                bones = id_data.bones

        return bones

class PoseBone(StructRNA, _GenericBone, metaclass=StructMetaPropGroup):
    __slots__ = ()

    @property
    def children(self):
        obj = self.id_data
        pbones = obj.pose.bones
        self_bone = self.bone

        return tuple(pbones[bone.name] for bone in obj.data.bones
                     if bone.parent == self_bone)

class Bone(StructRNA, _GenericBone, metaclass=StructMetaPropGroup):
    __slots__ = ()

class EditBone(StructRNA, _GenericBone, metaclass=StructMetaPropGroup):
    __slots__ = ()

    def align_orientation(self, other):
        """
        Align this bone to another by moving its tail and settings its roll
        the length of the other bone is not used.
        """
        vec = other.vector.normalized() * self.length
        self.tail = self.head + vec
        self.roll = other.roll

    def transform(self, matrix, scale=True, roll=True):
        """
        Transform the the bones head, tail, roll and envelope
        (when the matrix has a scale component).

        :arg matrix: 3x3 or 4x4 transformation matrix.
        :type matrix: :class:`mathutils.Matrix`
        :arg scale: Scale the bone envelope by the matrix.
        :type scale: bool
        :arg roll:

           Correct the roll to point in the same relative
           direction to the head and tail.

        :type roll: bool
        """
        from mathutils import Vector
        z_vec = self.matrix.to_3x3() @ Vector((0.0, 0.0, 1.0))
        self.tail = matrix @ self.tail
        self.head = matrix @ self.head

        if scale:
            scalar = matrix.median_scale
            self.head_radius *= scalar
            self.tail_radius *= scalar

        if roll:
            self.align_roll(matrix @ z_vec)

def ord_ind(i1, i2):
    if i1 < i2:
        return i1, i2
    return i2, i1

class Mesh(bpy_types.ID):
    __slots__ = ()

    def from_pydata(self, vertices, edges, faces):
        """
        Make a mesh from a list of vertices/edges/faces
        Until we have a nicer way to make geometry, use this.

        :arg vertices:

           float triplets each representing (X, Y, Z)
           eg: [(0.0, 1.0, 0.5), ...].

        :type vertices: iterable object
        :arg edges:

           int pairs, each pair contains two indices to the
           *vertices* argument. eg: [(1, 2), ...]

        :type edges: iterable object
        :arg faces:

           iterator of faces, each faces contains three or more indices to
           the *vertices* argument. eg: [(5, 6, 8, 9), (1, 2, 3), ...]

        :type faces: iterable object

        .. warning::

           Invalid mesh data
           *(out of range indices, edges with matching indices,
           2 sided faces... etc)* are **not** prevented.
           If the data used for mesh creation isn't known to be valid,
           run :class:`Mesh.validate` after this function.
        """
        from itertools import chain, islice, accumulate

        face_lengths = tuple(map(len, faces))

        self.vertices.add(len(vertices))
        self.edges.add(len(edges))
        self.loops.add(sum(face_lengths))
        self.polygons.add(len(faces))

        self.vertices.foreach_set("co", tuple(chain.from_iterable(vertices)))
        self.edges.foreach_set("vertices", tuple(chain.from_iterable(edges)))

        vertex_indices = tuple(chain.from_iterable(faces))
        loop_starts = tuple(islice(chain([0], accumulate(face_lengths)), len(faces)))

        self.polygons.foreach_set("loop_total", face_lengths)
        self.polygons.foreach_set("loop_start", loop_starts)
        self.polygons.foreach_set("vertices", vertex_indices)

        # if no edges - calculate them
        if faces and (not edges):
            self.update(calc_edges=True)
        elif edges:
            self.update(calc_edges_loose=True)

    @property
    def edge_keys(self):
        return [ed.key for ed in self.edges]

class MeshEdge(StructRNA):
    __slots__ = ()

    @property
    def key(self):
        return ord_ind(*tuple(self.vertices))

class MeshLoopTriangle(StructRNA):
    __slots__ = ()

    @property
    def center(self):
        """The midpoint of the face."""
        face_verts = self.vertices[:]
        mesh_verts = self.id_data.vertices
        return (
            mesh_verts[face_verts[0]].co +
            mesh_verts[face_verts[1]].co +
            mesh_verts[face_verts[2]].co
        ) / 3.0

    @property
    def edge_keys(self):
        verts = self.vertices[:]
        return (
            ord_ind(verts[0], verts[1]),
            ord_ind(verts[1], verts[2]),
            ord_ind(verts[2], verts[0]),
        )

class MeshPolygon(StructRNA):
    __slots__ = ()

    @property
    def edge_keys(self):
        verts = self.vertices[:]
        vlen = len(self.vertices)
        return [ord_ind(verts[i], verts[(i + 1) % vlen]) for i in range(vlen)]

    @property
    def loop_indices(self):
        start = self.loop_start
        end = start + self.loop_total
        return range(start, end)

class Text(bpy_types.ID):
    __slots__ = ()

    def as_string(self):
        """Return the text as a string."""
        return "\n".join(line.body for line in self.lines)

    def from_string(self, string):
        """Replace text with this string."""
        self.clear()
        self.write(string)

    def as_module(self):
        from os.path import splitext
        from types import ModuleType
        mod = ModuleType(splitext(self.name)[0])
        # TODO: We could use Text.compiled (C struct member)
        # if this is called often it will be much faster.
        exec(self.as_string(), mod.__dict__)
        return mod

class Sound(bpy_types.ID):
    __slots__ = ()

    @property
    def factory(self):
        """The aud.Factory object of the sound."""
        import aud
        return aud._sound_from_pointer(self.as_pointer())

class RNAMeta(type):
    # TODO(campbell): move to C-API
    @property
    def is_registered(cls):
        return "bl_rna" in cls.__dict__

class RNAMetaPropGroup(StructMetaPropGroup, RNAMeta):
    pass

# Same as 'Operator'
# only without 'as_keywords'
class Gizmo(StructRNA):
    __slots__ = ()

    def __getattribute__(self, attr):
        properties = StructRNA.path_resolve(self, "properties")
        bl_rna = getattr(properties, "bl_rna", None)
        if (bl_rna is not None) and (attr in bl_rna.properties):
            return getattr(properties, attr)
        return super().__getattribute__(attr)

    def __setattr__(self, attr, value):
        properties = StructRNA.path_resolve(self, "properties")
        bl_rna = getattr(properties, "bl_rna", None)
        if (bl_rna is not None) and (attr in bl_rna.properties):
            return setattr(properties, attr, value)
        return super().__setattr__(attr, value)

    def __delattr__(self, attr):
        properties = StructRNA.path_resolve(self, "properties")
        bl_rna = getattr(properties, "bl_rna", None)
        if (bl_rna is not None) and (attr in bl_rna.properties):
            return delattr(properties, attr)
        return super().__delattr__(attr)

    from _bpy import (
        _rna_gizmo_target_set_handler as target_set_handler,
        _rna_gizmo_target_get_value as target_get_value,
        _rna_gizmo_target_set_value as target_set_value,
        _rna_gizmo_target_get_range as target_get_range,
    )

    # Convenience wrappers around private `_gpu` module.
    def draw_custom_shape(self, shape, *, matrix=None, select_id=None):
        """
        Draw a shape created form :class:`bpy.types.Gizmo.draw_custom_shape`.

        :arg shape: The cached shape to draw.
        :type shape: Undefined.
        :arg matrix: 4x4 matrix, when not given
           :class:`bpy.types.Gizmo.matrix_world` is used.
        :type matrix: :class:`mathutils.Matrix`
        :arg select_id: The selection id.
           Only use when drawing within :class:`bpy.types.Gizmo.draw_select`.
        :type select_it: int
        """
        import gpu

        if matrix is None:
            matrix = self.matrix_world

        batch, shader = shape
        shader.bind()

        if select_id is not None:
            gpu.select.load_id(select_id)
        else:
            if self.is_highlight:
                color = (*self.color_highlight, self.alpha_highlight)
            else:
                color = (*self.color, self.alpha)
            shader.uniform_float("color", color)

        with gpu.matrix.push_pop():
            gpu.matrix.multiply_matrix(matrix)
            batch.draw()

    @staticmethod
    def new_custom_shape(type, verts):
        """
        Create a new shape that can be passed to :class:`bpy.types.Gizmo.draw_custom_shape`.

        :arg type: The type of shape to create in (POINTS, LINES, TRIS, LINE_STRIP).
        :type type: string
        :arg verts: Coordinates.
        :type verts: sequence of of 2D or 3D coordinates.
        :arg display_name: Optional callback that takes the full path, returns the name to display.
        :type display_name: Callable that takes a string and returns a string.
        :return: The newly created shape.
        :rtype: Undefined (it may change).
        """
        import gpu
        from gpu.types import (
            GPUBatch,
            GPUVertBuf,
            GPUVertFormat,
        )
        dims = len(verts[0])
        if dims not in {2, 3}:
            raise ValueError("Expected 2D or 3D vertex")
        fmt = GPUVertFormat()
        pos_id = fmt.attr_add(id="pos", comp_type='F32', len=dims, fetch_mode='FLOAT')
        vbo = GPUVertBuf(len=len(verts), format=fmt)
        vbo.attr_fill(id=pos_id, data=verts)
        batch = GPUBatch(type=type, buf=vbo)
        shader = gpu.shader.from_builtin('3D_UNIFORM_COLOR' if dims == 3 else '2D_UNIFORM_COLOR')
        batch.program_set(shader)
        return (batch, shader)

# Dummy class to keep the reference in `bpy_types_dict` and avoid
# erros like: "TypeError: expected GizmoGroup subclass of class ..."
class GizmoGroup(StructRNA):
    __slots__ = ()

# Only defined so operators members can be used by accessing self.order
# with doc generation 'self.properties.bl_rna.properties' can fail
class Operator(StructRNA, metaclass=RNAMeta):
    __slots__ = ()

    def __getattribute__(self, attr):
        properties = StructRNA.path_resolve(self, "properties")
        bl_rna = getattr(properties, "bl_rna", None)
        if (bl_rna is not None) and (attr in bl_rna.properties):
            return getattr(properties, attr)
        return super().__getattribute__(attr)

    def __setattr__(self, attr, value):
        properties = StructRNA.path_resolve(self, "properties")
        bl_rna = getattr(properties, "bl_rna", None)
        if (bl_rna is not None) and (attr in bl_rna.properties):
            return setattr(properties, attr, value)
        return super().__setattr__(attr, value)

    def __delattr__(self, attr):
        properties = StructRNA.path_resolve(self, "properties")
        bl_rna = getattr(properties, "bl_rna", None)
        if (bl_rna is not None) and (attr in bl_rna.properties):
            return delattr(properties, attr)
        return super().__delattr__(attr)

    def as_keywords(self, ignore=()):
        """Return a copy of the properties as a dictionary"""
        ignore = ignore + ("rna_type",)
        return {attr: getattr(self, attr)
                for attr in self.properties.rna_type.properties.keys()
                if attr not in ignore}

class Macro(StructRNA):
    # bpy_types is imported before ops is defined
    # so we have to do a local import on each run
    __slots__ = ()

    @classmethod
    def define(self, opname):
        from _bpy import ops
        return ops.macro_define(self, opname)

class PropertyGroup(StructRNA, metaclass=RNAMetaPropGroup):
    __slots__ = ()

class RenderEngine(StructRNA, metaclass=RNAMeta):
    __slots__ = ()

class KeyingSetInfo(StructRNA, metaclass=RNAMeta):
    __slots__ = ()

class AddonPreferences(StructRNA, metaclass=RNAMeta):
    __slots__ = ()

class _GenericUI:
    __slots__ = ()

    @classmethod
    def _dyn_ui_initialize(cls):
        draw_funcs = getattr(cls.draw, "_draw_funcs", None)

        if draw_funcs is None:

            def draw_ls(self, context):
                # ensure menus always get default context
                operator_context_default = self.layout.operator_context

                # Support filtering out by owner
                workspace = context.workspace
                if workspace.use_filter_by_owner:
                    owner_names = {owner_id.name for owner_id in workspace.owner_ids}
                else:
                    owner_names = None

                for func in draw_ls._draw_funcs:

                    # Begin 'owner_id' filter.
                    if owner_names is not None:
                        owner_id = getattr(func, "_owner", None)
                        if owner_id is not None:
                            if func._owner not in owner_names:
                                continue
                    # End 'owner_id' filter.

                    # so bad menu functions don't stop
                    # the entire menu from drawing
                    try:
                        func(self, context)
                    except:
                        import traceback
                        traceback.print_exc()

                    self.layout.operator_context = operator_context_default

            draw_funcs = draw_ls._draw_funcs = [cls.draw]
            cls.draw = draw_ls

        return draw_funcs

    @staticmethod
    def _dyn_owner_apply(draw_func):
        from _bpy import _bl_owner_id_get
        owner_id = _bl_owner_id_get()
        if owner_id is not None:
            draw_func._owner = owner_id

    @classmethod
    def is_extended(cls):
        return bool(getattr(cls.draw, "_draw_funcs", None))

    @classmethod
    def append(cls, draw_func):
        """
        Append a draw function to this menu,
        takes the same arguments as the menus draw function
        """
        draw_funcs = cls._dyn_ui_initialize()
        cls._dyn_owner_apply(draw_func)
        draw_funcs.append(draw_func)

    @classmethod
    def prepend(cls, draw_func):
        """
        Prepend a draw function to this menu, takes the same arguments as
        the menus draw function
        """
        draw_funcs = cls._dyn_ui_initialize()
        cls._dyn_owner_apply(draw_func)
        draw_funcs.insert(0, draw_func)

    @classmethod
    def remove(cls, draw_func):
        """Remove a draw function that has been added to this menu"""
        draw_funcs = cls._dyn_ui_initialize()
        try:
            draw_funcs.remove(draw_func)
        except ValueError:
            pass

class Panel(StructRNA, _GenericUI, metaclass=RNAMeta):
    __slots__ = ()

class UIList(StructRNA, _GenericUI, metaclass=RNAMeta):
    __slots__ = ()

class Header(StructRNA, _GenericUI, metaclass=RNAMeta):
    __slots__ = ()

class Menu(StructRNA, _GenericUI, metaclass=RNAMeta):
    __slots__ = ()

    def path_menu(self, searchpaths, operator, *,
                  props_default=None, prop_filepath="filepath",
                  filter_ext=None, filter_path=None, display_name=None,
                  add_operator=None):
        """
        Populate a menu from a list of paths.

        :arg searchpaths: Paths to scan.
        :type searchpaths: sequence of strings.
        :arg operator: The operator id to use with each file.
        :type operator: string
        :arg prop_filepath: Optional operator filepath property (defaults to "filepath").
        :type prop_filepath: string
        :arg props_default: Properties to assign to each operator.
        :type props_default: dict
        :arg filter_ext: Optional callback that takes the file extensions.

           Returning false excludes the file from the list.

        :type filter_ext: Callable that takes a string and returns a bool.
        :arg display_name: Optional callback that takes the full path, returns the name to display.
        :type display_name: Callable that takes a string and returns a string.
        """

        layout = self.layout

        import os
        import bpy.utils

        layout = self.layout

        if not searchpaths:
            layout.label(text="* Missing Paths *")

        # collect paths
        files = []
        for directory in searchpaths:
            files.extend([
                (f, os.path.join(directory, f))
                 for f in os.listdir(directory)
                 if (not f.startswith("."))
                 if ((filter_ext is None) or
                     (filter_ext(os.path.splitext(f)[1])))
                 if ((filter_path is None) or
                     (filter_path(f)))
            ])

        files.sort()

        col = layout.column(align=True)

        for f, filepath in files:
            # Intentionally pass the full path to 'display_name' callback,
            # since the callback may want to use part a directory in the name.
            row = col.row(align=True)
            name = display_name(filepath) if display_name else bpy.path.display_name(f)
            props = row.operator(
                operator,
                text=name,
                translate=False,
            )

            if props_default is not None:
                for attr, value in props_default.items():
                    setattr(props, attr, value)

            setattr(props, prop_filepath, filepath)
            if operator == "script.execute_preset":
                props.menu_idname = self.bl_idname

            if add_operator:
                props = row.operator(add_operator, text="", icon='REMOVE')
                props.name = name
                props.remove_name = True

        if add_operator:
            wm = bpy.data.window_managers[0]

            layout.separator()
            row = layout.row()

            sub = row.row()
            sub.emboss = 'NORMAL'
            sub.prop(wm, "preset_name", text="")

            props = row.operator(add_operator, text="", icon='ADD')
            props.name = wm.preset_name

    def draw_preset(self, _context):
        """
        Define these on the subclass:
        - preset_operator (string)
        - preset_subdir (string)

        Optionally:
        - preset_add_operator (string)
        - preset_extensions (set of strings)
        - preset_operator_defaults (dict of keyword args)
        """
        import bpy
        ext_valid = getattr(self, "preset_extensions", {".py", ".xml"})
        props_default = getattr(self, "preset_operator_defaults", None)
        add_operator = getattr(self, "preset_add_operator", None)
        self.path_menu(
            bpy.utils.preset_paths(self.preset_subdir),
            self.preset_operator,
            props_default=props_default,
            filter_ext=lambda ext: ext.lower() in ext_valid,
            add_operator=add_operator,
        )

    @classmethod
    def draw_collapsible(cls, context, layout):
        # helper function for (optionally) collapsed header menus
        # only usable within headers
        if context.area.show_menus:
            # Align menus to space them closely.
            layout.row(align=True).menu_contents(cls.__name__)
        else:
            layout.menu(cls.__name__, icon='COLLAPSEMENU')

class NodeTree(bpy_types.ID, metaclass=RNAMetaPropGroup):
    __slots__ = ()

class Node(StructRNA, metaclass=RNAMetaPropGroup):
    __slots__ = ()

    @classmethod
    def poll(cls, _ntree):
        return True

class NodeInternal(Node):
    __slots__ = ()

class NodeSocket(StructRNA, metaclass=RNAMetaPropGroup):
    __slots__ = ()

    @property
    def links(self):
        """List of node links from or to this socket. Warning: takes O(len(nodetree.links)) time."""
        return tuple(
            link for link in self.id_data.links
            if (link.from_socket == self or
                link.to_socket == self))

class NodeSocketInterface(StructRNA, metaclass=RNAMetaPropGroup):
    __slots__ = ()

# These are intermediate subclasses, need a bpy type too
class CompositorNode(NodeInternal):
    __slots__ = ()

    @classmethod
    def poll(cls, ntree):
        return ntree.bl_idname == 'CompositorNodeTree'

    def update(self):
        self.tag_need_exec()

class ShaderNode(NodeInternal):
    __slots__ = ()

    @classmethod
    def poll(cls, ntree):
        return ntree.bl_idname == 'ShaderNodeTree'

class TextureNode(NodeInternal):
    __slots__ = ()

    @classmethod
    def poll(cls, ntree):
        return ntree.bl_idname == 'TextureNodeTree'


import rospy

from yaw_controller import YawController
from pid import PID
from lowpass import LowPassFilter

GAS_DENSITY = 2.858
ONE_MPH = 0.44704

class Controller(object):
    def __init__(self, wheel_base, steer_ratio, max_lat_accel, max_steer_angle, accel_limit, decel_limit, vehicle_mass, fuel_capacity, brake_deadband,wheel_radius):

	self.yaw_controller = YawController(wheel_base, steer_ratio, 0.1, max_lat_accel, max_steer_angle)
	kp = 0.3
	ki = 0.1
	kd = 0.
	mn = 0.
	mx = 0.2
	self.throttle_controller = PID(kp,ki,kd,mn,mx)

	tau = 0.5 #1/(2pi*tau) cutoff frequency
	ts = 0.02 #sample_time
	self.vel_lpf = LowPassFilter(tau, ts)

	self.vehicle_mass = vehicle_mass
	self.fuel_capacity = fuel_capacity
	self.brake_deadband = brake_deadband
	self.decel_limit = decel_limit
	self.accel_limit = accel_limit
	self.wheel_radius = wheel_radius

	self.last_time = rospy.get_time()

    def control(self, current_vel, dbw_enabled, linear_vel,angular_vel):

        # TODO: Change the arg, kwarg list to suit your needs
        # Return throttle, brake, steer

	if not dbw_enabled:
	    self.throttle_controller.reset()
	    return 0.0, 0., 0.

	current_vel = self.vel_lpf.filt(current_vel)

	# rospy.logwarn("Angular vel : {0}".format(angular_vel))
	# rospy.logwarn("target vel : {0}".format(linear_vel))
	# rospy.logwarn("current vel : {0}".format(current_vel))

	steering = self.yaw_controller.get_steering(linear_vel, angular_vel, current_vel)

	vel_error = linear_vel - current_vel
	self.last_vel = current_vel

	current_time = rospy.get_time()
	sample_time = current_time - self.last_time
	self.last_time = current_time

	throttle = self.throttle_controller.step(vel_error, sample_time)
	brake = 0

	if linear_vel == 0. and current_vel < 1.0:
	   throttle = 0
	   brake = 400  #N*m to hold the car when we stop at light
	elif throttle < 0.1 and vel_error < 0:
	   throttle = 0
 	   decel = max(vel_error, self.decel_limit)
	   brake = abs(decel)*self.vehicle_mass*self.wheel_radius

	return throttle, brake, steering


# Rewrite the distance function from the chapter titled Fruitful functions
# so that it takes two Points as parameters instead of four numbers.

class Point:
    """ Point class represents and manipulates x,y coords. """

    def __init__(self, x=0, y=0):
        """ Create a new point at the origin """
        self.x = x
        self.y = y

    def __str__(self):
        return "({0}, {1})".format(self.x, self.y)

    def distance_from_origin(self):
        """ Compute my distance from the origin """
        return ((self.x ** 2) + (self.y ** 2)) ** 0.5

    def halfway(self, target):
        """ Return the halfway point between myself and the target """
        mx = (self.x + target.x) / 2
        my = (self.y + target.y) / 2
        return Point(mx, my)

#  Add a method reflect_x to Point which returns a new Point,
#  one which is the reflection of the point about the x-axis. For example, Point(3, 5).reflect_x() is (3, -5)

    def reflect_x(self):
        refx = (self.x)
        refy = -(self.y)
        return Point(refx, refy)

# Rewrite the distance function from the chapter titled Fruitful functions
    # so that it takes two Points as parameters instead of four numbers.
    def distance(p1, p2):
        dx = p2.x - p1.x
        dy = p2.y - p1.y
        dsquared = dx*dx + dy*dy
        result = dsquared**0.5
        return result

# Add a method slope_from_origin which returns the slope of the line joining the origin to the point. For example,
# Point(4, 10).slope_from_origin()
# 2.5

    def slope_from_origin(self):
        return self.y/self.x
# What cases will cause this method to fail? -> division by zero

# The equation of a straight line is “y = ax + b”, (or perhaps “y = mx + c”).
# The coefficients a and b completely describe the line.
# Write a method in the Point class so that if a point instance is given another point,
# it will compute the equation of the straight line joining the two points.
# It must return the two coefficients as a tuple of two values. For example,
# print(Point(4, 11).get_line_to(Point(6, 15)))
# (2, 3)

    def get_line_to(self, target):
        mx = target.x-self.x
        my = target.y-self.y
        slope = my/mx
        b = self.y - (slope * self.x)
        return (slope, b)

print(Point(4, 11).get_line_to(Point(6, 15)))


from proteus import Context
from proteus import Comm
comm = Comm.get()
ctx = Context.get()

# simulation flags for error analysis
#
# simFlagsList is initialized in proteus.iproteus
#
simFlagsList[0]['errorQuantities']=['u']
simFlagsList[0]['errorTypes']= ['numericalSolution'] #compute error in soln and glob. mass bal
simFlagsList[0]['errorNorms']= ['L2','H1'] #compute L2 norm in space or H0 or ...
simFlagsList[0]['errorTimes']= ['Last'] #'All', 'Last'
simFlagsList[0]['echo']=True

#
start
quit