#!/usr/bin/env python3

import pathlib
import fileinput

from ci.util import (
    check_env,
    existing_file,
)

repo_dir = check_env('REPO_DIR')
effective_version = check_env('EFFECTIVE_VERSION')

template_file = existing_file(pathlib.Path(repo_dir, 'concourse', 'resources', 'defaults.mako'))

lines_replaced = 0
string_to_match = 'tag = '

for line in fileinput.FileInput(str(template_file), inplace=True):
    if string_to_match in line:
        if lines_replaced != 0:
            raise RuntimeError(f'More than one image tag found in template file')
        leading_spaces = line.index(string_to_match)
        print(f'{leading_spaces * " "}{string_to_match}"{effective_version}"')
        lines_replaced = 1
    else:
        print(line, end='')


from oslo_log import log

from datetime import datetime
from flask import request, jsonify
from flask_restful import Resource, fields, marshal_with, abort

from clapton import db
from clapton.db.sqlalchemy import models

from clapton.api import types

LOG = log.getLogger(__name__)

class OrderList(Resource):
    def get(self):
        orders = db.get_session().query(models.Order).all()
        return jsonify((types.Order(many=True).dump(orders)).data), 200, {'X-Pagination-Total-Count': 1000}

        '''
        response = flask.make_response('[{"id": 123}]', 200)
        response.headers.extend({'X-Pagination-Total-Count': 1000,
                                 'Content-Type': 'application/json; charset=utf-8'})
        return response
        '''
        return [{"id": 123}], 200, {'X-Pagination-Total-Count': 1000}

    def post(self):
        '''
        validate request
        parser = reqparse.RequestParser()
        parser.add_argument(
            'total_amount',
            dest='total_amount',
            type=str,
            location='form',  # form, args, headers, cookies, json, files
            required=True,
            help='The orders\'s total amount',
        )
        args = parser.parse_args(strict=True)
        LOG.debug(args)
        LOG.debug(args.total_amount)
        return {}, 201
        '''
        data = request.get_json()
        if not data:
            return jsonify({'message': 'No imput data provided'}), 400
        data, errors = types.Order().load(data)
        if errors:
            return jsonify(errors), 422
        o = models.Order(id=data['id'])
        return jsonify((types.Order().dump(o)).data), 201

class Order(Resource):
    @marshal_with({'id': fields.String, 'created_at': fields.DateTime, 'links': fields.Nested({'items': fields.Url('items', absolute=True)})})
    def get(self, order_id):
        '''
        outputing format
        '''
        return {'id': 123, 'created_at': datetime.now(), 'links': []}

    def put(self, order_id):
        LOG.debug(request.form)
        LOG.debug(request.json)
        LOG.debug(dir(request))
        return {}, 201

    def delete(self, order_id):
        abort(500)
        # raise ValueError('haha')
        # raise werkzeug.exceptions.HTTPException(500)
        # raise werkzeug.exceptions.InternalServerError
        return '', 204

class OrderItemList(Resource):
    def get(self, order_id):
        return []


# Copyright 2019 Huawei Technologies Co.,LTD.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from unittest import mock

from oslo_db import exception as db_exc

from cyborg.common import exception
from cyborg import objects
from cyborg.tests.unit.db.base import DbTestCase
from cyborg.tests.unit import fake_deployable
from cyborg.tests.unit import fake_device
from cyborg.tests.unit.objects import test_objects

class TestDeployableObject(DbTestCase):

    @property
    def fake_device(self):
        db_device = fake_device.get_fake_devices_as_dict()[2]
        return db_device

    @property
    def fake_deployable(self):
        db_deploy = fake_deployable.fake_db_deployable(id=1)
        return db_deploy

    @property
    def fake_deployable2(self):
        db_deploy = fake_deployable.fake_db_deployable(id=2)
        return db_deploy

    def test_create(self):
        db_device = self.fake_device
        device = objects.Device(context=self.context,
                                **db_device)
        device.create(self.context)
        device_get = objects.Device.get(self.context, device.uuid)
        db_dpl = self.fake_deployable
        dpl = objects.Deployable(context=self.context,
                                 **db_dpl)

        dpl.device_id = device_get.id
        dpl.create(self.context)

        self.assertEqual(db_dpl['uuid'], dpl.uuid)

    def test_get(self):
        db_device = self.fake_device
        device = objects.Device(context=self.context,
                                **db_device)
        device.create(self.context)
        device_get = objects.Device.get(self.context, device.uuid)
        db_dpl = self.fake_deployable
        dpl = objects.Deployable(context=self.context,
                                 **db_dpl)

        dpl.device_id = device_get.id
        dpl.create(self.context)
        dpl_get = objects.Deployable.get(self.context, dpl.uuid)
        self.assertEqual(dpl_get.uuid, dpl.uuid)

    def test_get_by_filter(self):
        db_device = self.fake_device
        device = objects.Device(context=self.context,
                                **db_device)
        device.create(self.context)
        device_get = objects.Device.get(self.context, device.uuid)
        db_dpl = self.fake_deployable
        dpl = objects.Deployable(context=self.context,
                                 **db_dpl)

        dpl.device_id = device_get.id
        dpl.create(self.context)
        query = {"uuid": dpl['uuid']}
        dpl_get_list = objects.Deployable.get_by_filter(self.context, query)

        self.assertEqual(dpl_get_list[0].uuid, dpl.uuid)

    def test_save(self):
        db_device = self.fake_device
        device = objects.Device(context=self.context,
                                **db_device)
        device.create(self.context)
        device_get = objects.Device.get(self.context, device.uuid)
        db_dpl = self.fake_deployable
        dpl = objects.Deployable(context=self.context,
                                 **db_dpl)

        dpl.device_id = device_get.id
        dpl.create(self.context)
        dpl.num_accelerators = 8
        dpl.save(self.context)
        dpl_get = objects.Deployable.get(self.context, dpl.uuid)
        self.assertEqual(dpl_get.num_accelerators, 8)

    def test_destroy(self):
        db_device = self.fake_device
        device = objects.Device(context=self.context,
                                **db_device)
        device.create(self.context)
        device_get = objects.Device.get(self.context, device.uuid)
        db_dpl = self.fake_deployable
        dpl = objects.Deployable(context=self.context,
                                 **db_dpl)

        dpl.device_id = device_get.id
        dpl.create(self.context)
        self.assertEqual(db_dpl['uuid'], dpl.uuid)
        dpl.destroy(self.context)
        self.assertRaises(exception.ResourceNotFound,
                          objects.Deployable.get, self.context,
                          dpl.uuid)

class TestDeployableObject(test_objects._LocalTest,
                           TestDeployableObject):
    def _test_save_objectfield_fk_constraint_fails(self, foreign_key,
                                                   expected_exception):

        error = db_exc.DBReferenceError('table', 'constraint', foreign_key,
                                        'key_table')
        # Prevent lazy-loading any fields, results in InstanceNotFound
        deployable = fake_deployable.fake_deployable_obj(self.context)
        fields_with_save_methods = [field for field in deployable.fields
                                    if hasattr(deployable, '_save_%s' % field)]
        for field in fields_with_save_methods:
            @mock.patch.object(deployable, '_save_%s' % field)
            @mock.patch.object(deployable, 'obj_attr_is_set')
            def _test(mock_is_set, mock_save_field):
                mock_is_set.return_value = True
                mock_save_field.side_effect = error
                deployable.obj_reset_changes(fields=[field])
                deployable._changed_fields.add(field)
                self.assertRaises(expected_exception, deployable.save)
                deployable.obj_reset_changes(fields=[field])
            _test()


# from django.test import TestCase
# from  polls.models import Question,Choice
# from django.utils import timezone
#
#
# # from django.test import Client
# # Create your tests here.
# #用来写测试用例
# # model测试
# class StudyTestCsse(TestCase):
#     def setUp(self):
#         Question.objects.create(id=1,question_text="你的女朋友是谁?",pub_date=timezone.now())
#
#     def test_01(self):
#         u'''测试查询问题'''
#         question = Question.objects.get(id=1)
#         self.assertIn("你的女朋友是谁?",question.question_text)
#
#     def test_02(self):
#         u'''测试创建问题'''
#         Question.objects.create(id=2,question_text="今天吃什么?",pub_date=timezone.now())
#         question = Question.objects.get(id=2)
#         self.assertIn("今天吃什么",question.question_text)
#
#     def test_03(self):
#         u'''测试更新数据'''
#         question = Question.objects.get(id=1)
#         Question.objects.filter(id=1).update(question_text="周末是否加班")
#         question = Question.objects.get(id=1)
#         self.assertIn("周末是否加班",question.question_text)
#
#     def test_04(self):
#         u'''测试删除数据'''
#         question = Question.objects.get(id=1)
#         Question.objects.filter(id=1).delete()
#         self.assertEqual(0,len(Question.objects.all()))
#
# class choiceTestcase(TestCase):
#
#     def setUp(self):
#         Question.objects.create(id=1,question_text="what's new?",pub_date=timezone.now())
#         Choice.objects.create(id=1,choice_text='Not Much',votes=0,question_id=1)
#         Choice.objects.create(id=2,choice_text='The sky',votes=0,question_id=1)
#
#     def test_choice_query(self):
#         u'''测试问题选项查询'''
#         choice = Choice.objects.get(id=1)
#         self.assertEqual(choice.choice_text,"Not Much")


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models
import django.utils.timezone
from django.conf import settings
import model_utils.fields

class Migration(migrations.Migration):

    dependencies = [
        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
    ]

    operations = [
        migrations.CreateModel(
            name='Dataset',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, verbose_name='created', editable=False)),
                ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, verbose_name='modified', editable=False)),
                ('dataset', models.FileField(upload_to=b'datasets')),
                ('dimensions', models.PositiveIntegerField(default=0)),
                ('length', models.PositiveIntegerField(default=0)),
                ('filesize', models.PositiveIntegerField(default=0)),
                ('signature', models.CharField(unique=True, max_length=44, blank=True)),
                ('datatype', models.CharField(default=b'csv', max_length=4, choices=[(b'csv', b'csv'), (b'json', b'json'), (b'xml', b'xml')])),
                ('delimiter', models.CharField(default=b',', max_length=1)),
                ('uploader', models.ForeignKey(related_name='datasets', to=settings.AUTH_USER_MODEL)),
            ],
            options={
                'ordering': ('-created',),
                'db_table': 'datasets',
                'get_latest_by': 'created',
            },
        ),
    ]


# Copyright 2000-2002 by Andrew Dalke.
# Revisions copyright 2007-2008 by Peter Cock.
# All rights reserved.
# This code is part of the Biopython distribution and governed by its
# license.  Please see the LICENSE file that should have been included
# as part of this package.

"""Alphabets used in Seq objects etc to declare sequence type and letters.

This is used by sequences which contain a finite number of similar words.
"""

class Alphabet:
    size = None     # no fixed size for words
    letters = None  # no fixed alphabet; implement as a list-like
                    # interface,
    def __repr__(self):
        return self.__class__.__name__ + "()"

    def contains(self, other):
        """Does this alphabet 'contain' the other (OBSOLETE?).

        Returns a boolean.  This relies on the Alphabet subclassing
        hierarchy only, and does not check the letters property.
        This isn't ideal, and doesn't seem to work as intended
        with the AlphabetEncoder classes."""
        return isinstance(other, self.__class__)

    def _case_less(self):
        """Return an case-less variant of the current alphabet (PRIVATE)."""
        #TODO - remove this method by dealing with things in subclasses?
        if isinstance(self, ProteinAlphabet):
            return generic_protein
        elif isinstance(self, DNAAlphabet):
            return generic_dna
        elif isinstance(self, NucleotideAlphabet):
            return generic_rna
        elif isinstance(self, NucleotideAlphabet):
            return generic_nucleotide
        elif isinstance(self, SingleLetterAlphabet):
            return single_letter_alphabet
        else:
            return generic_alphabet

    def _upper(self):
        """Return an upper case variant of the current alphabet (PRIVATE)."""
        if not self.letters or self.letters==self.letters.upper():
            #Easy case, no letters or already upper case!
            return self
        else:
            #TODO - Raise NotImplementedError and handle via subclass?
            return self._case_less()

    def _lower(self):
        """Return a lower case variant of the current alphabet (PRIVATE)."""
        if not self.letters or self.letters==self.letters.lower():
            #Easy case, no letters or already lower case!
            return self
        else:
            #TODO - Raise NotImplementedError and handle via subclass?
            return self._case_less()

generic_alphabet = Alphabet()

class SingleLetterAlphabet(Alphabet):
    size = 1
    letters = None   # string of all letters in the alphabet

single_letter_alphabet = SingleLetterAlphabet()

########### Protein

class ProteinAlphabet(SingleLetterAlphabet):
    pass

generic_protein = ProteinAlphabet()

########### DNA
class NucleotideAlphabet(SingleLetterAlphabet):
    pass

generic_nucleotide = NucleotideAlphabet()

class DNAAlphabet(NucleotideAlphabet):
    pass

generic_dna = DNAAlphabet()

########### RNA

class RNAAlphabet(NucleotideAlphabet):
    pass

generic_rna = RNAAlphabet()

########### Other per-sequence encodings

class SecondaryStructure(SingleLetterAlphabet):
    letters = "HSTC"

class ThreeLetterProtein(Alphabet):
    size = 3
    letters = [
        "Ala", "Asx", "Cys", "Asp", "Glu", "Phe", "Gly", "His", "Ile",
        "Lys", "Leu", "Met", "Asn", "Pro", "Gln", "Arg", "Ser", "Thr",
        "Sec", "Val", "Trp", "Xaa", "Tyr", "Glx",
        ]

###### Non per-sequence modifications

# (These are Decorator classes)

class AlphabetEncoder:
    def __init__(self, alphabet, new_letters):
        self.alphabet = alphabet
        self.new_letters = new_letters
        if alphabet.letters is not None:
            self.letters = alphabet.letters + new_letters
        else:
            self.letters = None
    def __getattr__(self, key):
        if key[:2] == "__" and key[-2:] == "__":
            raise AttributeError(key)
        return getattr(self.alphabet, key)

    def __repr__(self):
        return "%s(%r, %r)" % (self.__class__.__name__, self.alphabet,
                               self.new_letters)

    def contains(self, other):
        """Does this alphabet 'contain' the other (OBSOLETE?).

        This is isn't implemented for the base AlphabetEncoder,
        which will always return 0 (False)."""
        return 0

    def _upper(self):
        """Return an upper case variant of the current alphabet (PRIVATE)."""
        return AlphabetEncoder(self.alphabet._upper(), self.new_letters.upper())

    def _lower(self):
        """Return a lower case variant of the current alphabet (PRIVATE)."""
        return AlphabetEncoder(self.alphabet._lower(), self.new_letters.lower())

class Gapped(AlphabetEncoder):
    def __init__(self, alphabet, gap_char = "-"):
        AlphabetEncoder.__init__(self, alphabet, gap_char)
        self.gap_char = gap_char

    def contains(self, other):
        """Does this alphabet 'contain' the other (OBSOLETE?).

        Returns a boolean.  This relies on the Alphabet subclassing
        hierarchy, and attempts to check the gap character.  This fails
        if the other alphabet does not have a gap character!
        """
        return other.gap_char == self.gap_char and \
               self.alphabet.contains(other.alphabet)

    def _upper(self):
        """Return an upper case variant of the current alphabet (PRIVATE)."""
        return Gapped(self.alphabet._upper(), self.gap_char.upper())

    def _lower(self):
        """Return a lower case variant of the current alphabet (PRIVATE)."""
        return Gapped(self.alphabet._lower(), self.gap_char.lower())

class HasStopCodon(AlphabetEncoder):
    def __init__(self, alphabet, stop_symbol = "*"):
        AlphabetEncoder.__init__(self, alphabet, stop_symbol)
        self.stop_symbol = stop_symbol

    def __cmp__(self, other):
        x = cmp(self.alphabet, other.alphabet)
        if x == 0:
            return cmp(self.stop_symbol, other.stop_symbol)
        return x

    def contains(self, other):
        """Does this alphabet 'contain' the other (OBSOLETE?).

        Returns a boolean.  This relies on the Alphabet subclassing
        hierarchy, and attempts to check the stop symbol.  This fails
        if the other alphabet does not have a stop symbol!
        """
        return other.stop_symbol == self.stop_symbol and \
               self.alphabet.contains(other.alphabet)

    def _upper(self):
        """Return an upper case variant of the current alphabet (PRIVATE)."""
        return HasStopCodon(self.alphabet._upper(), self.stop_symbol.upper())

    def _lower(self):
        """Return a lower case variant of the current alphabet (PRIVATE)."""
        return HasStopCodon(self.alphabet._lower(), self.stop_symbol.lower())

def _get_base_alphabet(alphabet):
    """Returns the non-gapped non-stop-codon Alphabet object (PRIVATE)."""
    a = alphabet
    while isinstance(a, AlphabetEncoder):
        a = a.alphabet
    assert isinstance(a, Alphabet), \
           "Invalid alphabet found, %s" % repr(a)
    return a

def _ungap(alphabet):
    """Returns the alphabet without any gap encoder (PRIVATE)."""
    #TODO - Handle via method of the objects?
    if not hasattr(alphabet, "gap_char"):
        return alphabet
    elif isinstance(alphabet, Gapped):
        return alphabet.alphabet
    elif isinstance(alphabet, HasStopCodon):
        return HasStopCodon(_ungap(alphabet.alphabet), stop_symbol=alphabet.stop_symbol)
    elif isinstance(alphabet, AlphabetEncoder):
        return AlphabetEncoder(_ungap(alphabet.alphabet), letters=alphabet.letters)
    else:
        raise NotImplementedError

def _consensus_base_alphabet(alphabets):
    """Returns a common but often generic base alphabet object (PRIVATE).

    This throws away any AlphabetEncoder information, e.g. Gapped alphabets.

    Note that DNA+RNA -> Nucleotide, and Nucleotide+Protein-> generic single
    letter.  These DO NOT raise an exception!"""
    common = None
    for alpha in alphabets:
        a = _get_base_alphabet(alpha)
        if common is None:
            common = a
        elif common == a:
            pass
        elif isinstance(a, common.__class__):
            pass
        elif isinstance(common, a.__class__):
            common = a
        elif isinstance(a, NucleotideAlphabet) \
        and isinstance(common, NucleotideAlphabet):
            #e.g. Give a mix of RNA and DNA alphabets
            common = generic_nucleotide
        elif isinstance(a, SingleLetterAlphabet) \
        and isinstance(common, SingleLetterAlphabet):
            #This is a pretty big mis-match!
            common = single_letter_alphabet
        else:
            #We have a major mis-match... take the easy way out!
            return generic_alphabet
    if common is None:
        #Given NO alphabets!
        return generic_alphabet
    return common

def _consensus_alphabet(alphabets):
    """Returns a common but often generic alphabet object (PRIVATE).

    Note that DNA+RNA -> Nucleotide, and Nucleotide+Protein-> generic single
    letter.  These DO NOT raise an exception!

    This is aware of Gapped and HasStopCodon and new letters added by
    other AlphabetEncoders.  This WILL raise an exception if more than
    one gap character or stop symbol is present."""
    base = _consensus_base_alphabet(alphabets)
    gap = None
    stop = None
    new_letters = ""
    for alpha in alphabets:
        #Gaps...
        if not hasattr(alpha, "gap_char"):
            pass
        elif gap is None:
            gap = alpha.gap_char
        elif gap == alpha.gap_char:
            pass
        else:
            raise ValueError("More than one gap character present")
        #Stops...
        if not hasattr(alpha, "stop_symbol"):
            pass
        elif stop is None:
            stop = alpha.stop_symbol
        elif stop == alpha.stop_symbol:
            pass
        else:
            raise ValueError("More than one stop symbol present")
        #New letters...
        if hasattr(alpha, "new_letters"):
            for letter in alpha.new_letters:
                if letter not in new_letters \
                and letter != gap and letter != stop:
                    new_letters += letter

    alpha = base
    if new_letters:
        alpha = AlphabetEncoder(alpha, new_letters)
    if gap:
        alpha = Gapped(alpha, gap_char=gap)
    if stop:
        alpha = HasStopCodon(alpha, stop_symbol=stop)
    return alpha

def _check_type_compatible(alphabets):
    """Returns True except for DNA+RNA or Nucleotide+Protein (PRIVATE).

    This relies on the Alphabet subclassing hierarchy.  It does not
    check things like gap characters or stop symbols."""
    dna, rna, nucl, protein = False, False, False, False
    for alpha in alphabets:
        a = _get_base_alphabet(alpha)
        if isinstance(a, DNAAlphabet):
            dna = True
            nucl = True
            if rna or protein : return False
        elif isinstance(a, RNAAlphabet):
            rna = True
            nucl = True
            if dna or protein : return False
        elif isinstance(a, NucleotideAlphabet):
            nucl = True
            if protein : return False
        elif isinstance(a, ProteinAlphabet):
            protein = True
            if nucl : return False
    return True


""" Setup remote debugger with Python Tools for Visual Studio (PTVSD)

"""
import os

from .celery_log_setup import get_task_logger

REMOTE_DEBUG_PORT = 3000

log = get_task_logger(__name__)

def setup_remote_debugging(force_enabled: bool = False, *, boot_mode=None) -> None:
    """ Programaticaly enables remote debugging if SC_BOOT_MODE==debug-ptvsd

    """
    if "SC_BOOT_MODE" not in os.environ:
        log.warning("Remote debugging only available when running in a container")
        return

    boot_mode = boot_mode or os.environ.get("SC_BOOT_MODE")

    if boot_mode == "debug-ptvsd" or force_enabled:
        try:
            log.debug("Enabling attach ptvsd ...")
            #
            # SEE https://github.com/microsoft/ptvsd#enabling-debugging
            #
            import ptvsd

            ptvsd.enable_attach(
                address=("0.0.0.0", REMOTE_DEBUG_PORT), redirect_output=True
            )  # nosec

        except ImportError:
            log.exception("Unable to use remote debugging. ptvsd is not installed")

        else:
            log.info("Remote debugging enabled: listening port %s", REMOTE_DEBUG_PORT)
    else:
        log.debug("Booting without remote debugging since SC_BOOT_MODE=%s", boot_mode)

__all__ = ["setup_remote_debugging"]


# -*- encoding: utf-8 -*-
from __future__ import unicode_literals

from slack_g_cal.parse import JSON, Datetime

class WitDatetimeContainer(JSON):
    """ Container wrapping datetime values from the Wit API """
    def __init__(self, **dt_json):
        self.is_interval = dt_json['type'] == 'interval'
        # Get rid of values; we don't need this parameter
        dt_json.pop('values', None)
        if self.is_interval:
            from_, to_ = dt_json.pop('from'), dt_json.pop('to')
            self.dt_from = WitDatetime(date_input=from_.value, grain=from_.grain)
            self.dt_to = WitDatetime(date_input=to_.value, grain=to_.grain)
        else:
            self.date = WitDatetime(date_input=dt_json.pop('value'), grain=dt_json.pop('grain'))
        super(WitDatetimeContainer, self).__init__(**dt_json)

class WitDatetime(Datetime):
    def __init__(self, date_input, **dt_json):
        self.grain = dt_json.pop('grain')
        super(WitDatetime, self).__init__(date_input=date_input, **dt_json)

    def adjust_grain_by(self, adj_val):
        kwargs = {self.grain: getattr(self._datetime, self.grain) + adj_val}
        self._datetime = self._datetime.replace(**kwargs)


# -*- coding: utf-8 -*-
"""IPython Test Suite Runner.

This module provides a main entry point to a user script to test IPython
itself from the command line. There are two ways of running this script:

1. With the syntax `iptest all`.  This runs our entire test suite by
   calling this script (with different arguments) recursively.  This
   causes modules and package to be tested in different processes, using nose
   or trial where appropriate.
2. With the regular nose syntax, like `iptest -vvs IPython`.  In this form
   the script simply calls nose, but with special command line flags and
   plugins loaded.

"""

# Copyright (c) IPython Development Team.
# Distributed under the terms of the Modified BSD License.

import glob
from io import BytesIO
import os
import os.path as path
import sys
from threading import Thread, Lock, Event
import warnings

import nose.plugins.builtin
from nose.plugins.xunit import Xunit
from nose import SkipTest
from nose.core import TestProgram
from nose.plugins import Plugin
from nose.util import safe_str

from IPython import version_info
from IPython.utils.py3compat import decode
from IPython.utils.importstring import import_item
from IPython.testing.plugin.ipdoctest import IPythonDoctest
from IPython.external.decorators import KnownFailure, knownfailureif

pjoin = path.join

# Enable printing all warnings raise by IPython's modules
warnings.filterwarnings('ignore', message='.*Matplotlib is building the font cache.*', category=UserWarning, module='.*')
warnings.filterwarnings('error', message='.*', category=ResourceWarning, module='.*')
warnings.filterwarnings('error', message=".*{'config': True}.*", category=DeprecationWarning, module='IPy.*')
warnings.filterwarnings('default', message='.*', category=Warning, module='IPy.*')

warnings.filterwarnings('error', message='.*apply_wrapper.*', category=DeprecationWarning, module='.*')
warnings.filterwarnings('error', message='.*make_label_dec', category=DeprecationWarning, module='.*')
warnings.filterwarnings('error', message='.*decorated_dummy.*', category=DeprecationWarning, module='.*')
warnings.filterwarnings('error', message='.*skip_file_no_x11.*', category=DeprecationWarning, module='.*')
warnings.filterwarnings('error', message='.*onlyif_any_cmd_exists.*', category=DeprecationWarning, module='.*')

warnings.filterwarnings('error', message='.*disable_gui.*', category=DeprecationWarning, module='.*')

warnings.filterwarnings('error', message='.*ExceptionColors global is deprecated.*', category=DeprecationWarning, module='.*')

# Jedi older versions
warnings.filterwarnings(
    'error', message='.*elementwise != comparison failed and.*', category=FutureWarning, module='.*')

if version_info < (6,):
    # nose.tools renames all things from `camelCase` to `snake_case` which raise an
    # warning with the runner they also import from standard import library. (as of Dec 2015)
    # Ignore, let's revisit that in a couple of years for IPython 6.
    warnings.filterwarnings(
        'ignore', message='.*Please use assertEqual instead', category=Warning, module='IPython.*')

if version_info < (7,):
    warnings.filterwarnings('ignore', message='.*Completer.complete.*',
                            category=PendingDeprecationWarning, module='.*')
else:
    warnings.warn(
        'Completer.complete was pending deprecation and should be changed to Deprecated', FutureWarning)

# ------------------------------------------------------------------------------
# Monkeypatch Xunit to count known failures as skipped.
# ------------------------------------------------------------------------------
def monkeypatch_xunit():
    try:
        knownfailureif(True)(lambda: None)()
    except Exception as e:
        KnownFailureTest = type(e)

    def addError(self, test, err, capt=None):
        if issubclass(err[0], KnownFailureTest):
            err = (SkipTest,) + err[1:]
        return self.orig_addError(test, err, capt)

    Xunit.orig_addError = Xunit.addError
    Xunit.addError = addError

#-----------------------------------------------------------------------------
# Check which dependencies are installed and greater than minimum version.
#-----------------------------------------------------------------------------
def extract_version(mod):
    return mod.__version__

def test_for(item, min_version=None, callback=extract_version):
    """Test to see if item is importable, and optionally check against a minimum
    version.

    If min_version is given, the default behavior is to check against the
    `__version__` attribute of the item, but specifying `callback` allows you to
    extract the value you are interested in. e.g::

        In [1]: import sys

        In [2]: from IPython.testing.iptest import test_for

        In [3]: test_for('sys', (2,6), callback=lambda sys: sys.version_info)
        Out[3]: True

    """
    try:
        check = import_item(item)
    except (ImportError, RuntimeError):
        # GTK reports Runtime error if it can't be initialized even if it's
        # importable.
        return False
    else:
        if min_version:
            if callback:
                # extra processing step to get version to compare
                check = callback(check)

            return check >= min_version
        else:
            return True

# Global dict where we can store information on what we have and what we don't
# have available at test run time
have = {'matplotlib': test_for('matplotlib'),
        'pygments': test_for('pygments'),
        'sqlite3': test_for('sqlite3')}

#-----------------------------------------------------------------------------
# Test suite definitions
#-----------------------------------------------------------------------------

test_group_names = ['core',
                    'extensions', 'lib', 'terminal', 'testing', 'utils',
                   ]

class TestSection(object):
    def __init__(self, name, includes):
        self.name = name
        self.includes = includes
        self.excludes = []
        self.dependencies = []
        self.enabled = True

    def exclude(self, module):
        if not module.startswith('IPython'):
            module = self.includes[0] + "." + module
        self.excludes.append(module.replace('.', os.sep))

    def requires(self, *packages):
        self.dependencies.extend(packages)

    @property
    def will_run(self):
        return self.enabled and all(have[p] for p in self.dependencies)

# Name -> (include, exclude, dependencies_met)
test_sections = {n:TestSection(n, ['IPython.%s' % n]) for n in test_group_names}

# Exclusions and dependencies
# ---------------------------

# core:
sec = test_sections['core']
if not have['sqlite3']:
    sec.exclude('tests.test_history')
    sec.exclude('history')
if not have['matplotlib']:
    sec.exclude('pylabtools'),
    sec.exclude('tests.test_pylabtools')

# lib:
sec = test_sections['lib']
sec.exclude('kernel')
if not have['pygments']:
    sec.exclude('tests.test_lexers')
# We do this unconditionally, so that the test suite doesn't import
# gtk, changing the default encoding and masking some unicode bugs.
sec.exclude('inputhookgtk')
# We also do this unconditionally, because wx can interfere with Unix signals.
# There are currently no tests for it anyway.
sec.exclude('inputhookwx')
# Testing inputhook will need a lot of thought, to figure out
# how to have tests that don't lock up with the gui event
# loops in the picture
sec.exclude('inputhook')

# testing:
sec = test_sections['testing']
# These have to be skipped on win32 because they use echo, rm, cd, etc.
# See ticket https://github.com/ipython/ipython/issues/87
if sys.platform == 'win32':
    sec.exclude('plugin.test_exampleip')
    sec.exclude('plugin.dtexample')

# don't run jupyter_console tests found via shim
test_sections['terminal'].exclude('console')

# extensions:
sec = test_sections['extensions']
# This is deprecated in favour of rpy2
sec.exclude('rmagic')
# autoreload does some strange stuff, so move it to its own test section
sec.exclude('autoreload')
sec.exclude('tests.test_autoreload')
test_sections['autoreload'] = TestSection('autoreload',
        ['IPython.extensions.autoreload', 'IPython.extensions.tests.test_autoreload'])
test_group_names.append('autoreload')

#-----------------------------------------------------------------------------
# Functions and classes
#-----------------------------------------------------------------------------

def check_exclusions_exist():
    from IPython.paths import get_ipython_package_dir
    from warnings import warn
    parent = os.path.dirname(get_ipython_package_dir())
    for sec in test_sections:
        for pattern in sec.exclusions:
            fullpath = pjoin(parent, pattern)
            if not os.path.exists(fullpath) and not glob.glob(fullpath + '.*'):
                warn("Excluding nonexistent file: %r" % pattern)

class ExclusionPlugin(Plugin):
    """A nose plugin to effect our exclusions of files and directories.
    """
    name = 'exclusions'
    score = 3000  # Should come before any other plugins

    def __init__(self, exclude_patterns=None):
        """
        Parameters
        ----------

        exclude_patterns : sequence of strings, optional
          Filenames containing these patterns (as raw strings, not as regular
          expressions) are excluded from the tests.
        """
        self.exclude_patterns = exclude_patterns or []
        super(ExclusionPlugin, self).__init__()

    def options(self, parser, env=os.environ):
        Plugin.options(self, parser, env)

    def configure(self, options, config):
        Plugin.configure(self, options, config)
        # Override nose trying to disable plugin.
        self.enabled = True

    def wantFile(self, filename):
        """Return whether the given filename should be scanned for tests.
        """
        if any(pat in filename for pat in self.exclude_patterns):
            return False
        return None

    def wantDirectory(self, directory):
        """Return whether the given directory should be scanned for tests.
        """
        if any(pat in directory for pat in self.exclude_patterns):
            return False
        return None

class StreamCapturer(Thread):
    daemon = True  # Don't hang if main thread crashes
    started = False
    def __init__(self, echo=False):
        super(StreamCapturer, self).__init__()
        self.echo = echo
        self.streams = []
        self.buffer = BytesIO()
        self.readfd, self.writefd = os.pipe()
        self.buffer_lock = Lock()
        self.stop = Event()

    def run(self):
        self.started = True

        while not self.stop.is_set():
            chunk = os.read(self.readfd, 1024)

            with self.buffer_lock:
                self.buffer.write(chunk)
            if self.echo:
                sys.stdout.write(decode(chunk))

        os.close(self.readfd)
        os.close(self.writefd)

    def reset_buffer(self):
        with self.buffer_lock:
            self.buffer.truncate(0)
            self.buffer.seek(0)

    def get_buffer(self):
        with self.buffer_lock:
            return self.buffer.getvalue()

    def ensure_started(self):
        if not self.started:
            self.start()

    def halt(self):
        """Safely stop the thread."""
        if not self.started:
            return

        self.stop.set()
        os.write(self.writefd, b'\0')  # Ensure we're not locked in a read()
        self.join()

class SubprocessStreamCapturePlugin(Plugin):
    name='subprocstreams'
    def __init__(self):
        Plugin.__init__(self)
        self.stream_capturer = StreamCapturer()
        self.destination = os.environ.get('IPTEST_SUBPROC_STREAMS', 'capture')
        # This is ugly, but distant parts of the test machinery need to be able
        # to redirect streams, so we make the object globally accessible.
        nose.iptest_stdstreams_fileno = self.get_write_fileno

    def get_write_fileno(self):
        if self.destination == 'capture':
            self.stream_capturer.ensure_started()
            return self.stream_capturer.writefd
        elif self.destination == 'discard':
            return os.open(os.devnull, os.O_WRONLY)
        else:
            return sys.__stdout__.fileno()

    def configure(self, options, config):
        Plugin.configure(self, options, config)
        # Override nose trying to disable plugin.
        if self.destination == 'capture':
            self.enabled = True

    def startTest(self, test):
        # Reset log capture
        self.stream_capturer.reset_buffer()

    def formatFailure(self, test, err):
        # Show output
        ec, ev, tb = err
        captured = self.stream_capturer.get_buffer().decode('utf-8', 'replace')
        if captured.strip():
            ev = safe_str(ev)
            out = [ev, '>> begin captured subprocess output <<',
                    captured,
                    '>> end captured subprocess output <<']
            return ec, '\n'.join(out), tb

        return err

    formatError = formatFailure

    def finalize(self, result):
        self.stream_capturer.halt()

def run_iptest():
    """Run the IPython test suite using nose.

    This function is called when this script is **not** called with the form
    `iptest all`.  It simply calls nose with appropriate command line flags
    and accepts all of the standard nose arguments.
    """
    # Apply our monkeypatch to Xunit
    if '--with-xunit' in sys.argv and not hasattr(Xunit, 'orig_addError'):
        monkeypatch_xunit()

    arg1 = sys.argv[1]
    if arg1 in test_sections:
        section = test_sections[arg1]
        sys.argv[1:2] = section.includes
    elif arg1.startswith('IPython.') and arg1[8:] in test_sections:
        section = test_sections[arg1[8:]]
        sys.argv[1:2] = section.includes
    else:
        section = TestSection(arg1, includes=[arg1])

    argv = sys.argv + [ '--detailed-errors',  # extra info in tracebacks
                        # We add --exe because of setuptools' imbecility (it
                        # blindly does chmod +x on ALL files).  Nose does the
                        # right thing and it tries to avoid executables,
                        # setuptools unfortunately forces our hand here.  This
                        # has been discussed on the distutils list and the
                        # setuptools devs refuse to fix this problem!
                        '--exe',
                        ]
    if '-a' not in argv and '-A' not in argv:
        argv = argv + ['-a', '!crash']

    if nose.__version__ >= '0.11':
        # I don't fully understand why we need this one, but depending on what
        # directory the test suite is run from, if we don't give it, 0 tests
        # get run.  Specifically, if the test suite is run from the source dir
        # with an argument (like 'iptest.py IPython.core', 0 tests are run,
        # even if the same call done in this directory works fine).  It appears
        # that if the requested package is in the current dir, nose bails early
        # by default.  Since it's otherwise harmless, leave it in by default
        # for nose >= 0.11, though unfortunately nose 0.10 doesn't support it.
        argv.append('--traverse-namespace')

    plugins = [ ExclusionPlugin(section.excludes), KnownFailure(),
               SubprocessStreamCapturePlugin() ]

    # we still have some vestigial doctests in core
    if (section.name.startswith(('core', 'IPython.core', 'IPython.utils'))):
        plugins.append(IPythonDoctest())
        argv.extend([
            '--with-ipdoctest',
            '--ipdoctest-tests',
            '--ipdoctest-extension=txt',
        ])

    # Use working directory set by parent process (see iptestcontroller)
    if 'IPTEST_WORKING_DIR' in os.environ:
        os.chdir(os.environ['IPTEST_WORKING_DIR'])

    # We need a global ipython running in this process, but the special
    # in-process group spawns its own IPython kernels, so for *that* group we
    # must avoid also opening the global one (otherwise there's a conflict of
    # singletons).  Ultimately the solution to this problem is to refactor our
    # assumptions about what needs to be a singleton and what doesn't (app
    # objects should, individual shells shouldn't).  But for now, this
    # workaround allows the test suite for the inprocess module to complete.
    if 'kernel.inprocess' not in section.name:
        from IPython.testing import globalipapp
        globalipapp.start_ipython()

    # Now nose can run
    TestProgram(argv=argv, addplugins=plugins)

if __name__ == '__main__':
    run_iptest()


#DateTimeExample1.py
from datetime import *

#Will print current date and time
print("Current Date Time : ",datetime.now())


import os
from io import StringIO
from django.contrib.gis.geos import Point
from django.test import TestCase
from uk_geo_utils.models import Onspd
from uk_geo_utils.management.commands.import_onspd import Command

class OnsudImportTest(TestCase):

    def test_import_onspd(self):
        # check table is empty before we start
        self.assertEqual(0, Onspd.objects.count())

        # path to file we're going to import
        csv_path = os.path.abspath(
            os.path.join(
                os.path.dirname(os.path.abspath(__file__)),
                '../fixtures/onspd'
            )
        )

        cmd = Command()

        # supress output
        out = StringIO()
        cmd.stdout = out

        # import data
        opts = {
            'path': csv_path,
        }
        cmd.handle(**opts)

        # ensure all our tasty data has been imported
        self.assertEqual(4, Onspd.objects.count())

        # row with valid grid ref should have valid Point() location
        al11aa = Onspd.objects.filter(pcds="AL1 1AA")[0]
        self.assertEqual(Point(-0.341337, 51.749084, srid=4326), al11aa.location)

        # row with invalid grid ref should have NULL location
        im11aa = Onspd.objects.filter(pcds="IM1 1AA")[0]
        self.assertIsNone(im11aa.location)


#!/usr/bin/env python
# Copyright 2019 Xilinx Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import json
import argparse
import shutil

arg_parser = argparse.ArgumentParser(description="This is a script to convert coco anntations to voc-like annotations.")
arg_parser.add_argument('-ti', '--train_images', type=str, default="./coco2014/train2014", help='where to put coco2014 train images.')
arg_parser.add_argument('-vi', '--val_images', type=str, default='./coco2014/val2014', help='where to put coco2014 val images.')
arg_parser.add_argument('-ta', '--train_anno', type=str, default='./coco2014/instances_train2014.json', help='where to put cooc2014 train set annotations.')
arg_parser.add_argument('-va', '--val_anno', type=str, default='./coco2014/instances_val2014.json', help='where to put coco2014 val set annotations')
arg_parser.add_argument('-tlf', '--tran_list_file', type=str, default='./coco2014/train2014.txt', help='image list for training')
arg_parser.add_argument('-vlf', '--val_list_file', type=str, default='./coco2014/val2014.txt', help='image list for evalution.')
arg_parser.add_argument('-ai', '--all_images', type=str, default='./coco2014/Images', help='where to put all images.')
arg_parser.add_argument('-aa', '--all_anno', type=str, default='./coco2014/Annotations', help='where to put all annotations.')
args = arg_parser.parse_args()

'''How to organize coco dataset folder:
 inputs:
 coco2014/
       |->train2014/
       |->val2014/
       |->instances_train2014.json
       |->instances_val2014.json

outputs:
 coco2014/
       |->Annotations/
       |->Images/
       |->train2014.txt
       |->val2014.txt
'''

def convert_images_coco2voc(args):
    assert os.path.exists(args.train_images)
    assert os.path.exists(args.val_images)
    os.system('mv ' + args.train_images + ' ' + args.all_images)
    imagename_list = os.listdir(args.val_images)
    for imagename in imagename_list:
       shutil.copy(os.path.join(args.val_images, imagename), args.all_images)
    os.system('rm -r ' + args.val_images)

def generate_cid_name(json_object):
    id2name_dict = {}
    for ind, category_info in enumerate(json_object['categories']):
        id2name_dict[category_info['id']] = category_info['name']
    return id2name_dict

def generate_image_dict(json_object):
    id2image_dict = {}
    for ind, image_info in enumerate(json_object['images']):
        id2image_dict[image_info['id']] = image_info['file_name']
    return id2image_dict

def generate_annotation_files(json_object, annotation_path, id2image_dict, id2name, image_list_file):
    if not os.path.exists(annotation_path):
        os.mkdir(annotation_path)
    f_image = open(image_list_file, 'w')
    all_images_name = []
    for ind, anno_info in enumerate(json_object['annotations']):
        print('preprocess: {}'.format(ind))
        category_id = anno_info['category_id']
        cls_name = id2name[category_id]
        if cls_name != "person":
            continue
        image_id = anno_info['image_id']
        image_name = id2image_dict[image_id]
        bbox = anno_info['bbox']
        bbox[2] = bbox[0] + bbox[2]
        bbox[3] = bbox[3] + bbox[1]
        bbox_str = ' '.join([str(int(x)) for x in bbox])
        with open(os.path.join(annotation_path, image_name.split('.')[0] + '.txt'), 'a') as f_anno:
            f_anno.writelines(image_name.split('.')[0] + " " + cls_name + " " + bbox_str + "\n")
        if image_name not in all_images_name:
            all_images_name.append(image_name)
    for image_name in all_images_name:
        f_image.writelines(image_name.split('.')[0] + "\n")
    f_image.close()

def convert_anno_coco2voc(coco_anno_file, image_list_file, all_anno_path):
    with open(coco_anno_file, 'r') as f_ann:
         line = f_ann.readlines()
    json_object = json.loads(line[0])
    id2name = generate_cid_name(json_object)
    id2image_dict = generate_image_dict(json_object)
    generate_annotation_files(json_object, all_anno_path, id2image_dict, id2name, image_list_file)

def convert_anno_all(args):
    convert_anno_coco2voc(args.train_anno, args.tran_list_file, args.all_anno)
    convert_anno_coco2voc(args.val_anno, args.val_list_file, args.all_anno)

if __name__  == "__main__":
    convert_anno_all(args)
    convert_images_coco2voc(args)


#!/usr/bin/env python
"""
models for the mailroom program.

This is where the program logic is.

This version has been made Object Oriented.
"""

# handy utility to make pretty printing easier
from textwrap import dedent
from pathlib import Path

import json_save.json_save_dec as js
import json

from . import data_dir

@js.json_save
class Donor:
    """
    class to hold the information about a single donor
    """
    name = js.String()
    donations = js.List()

    # reference to the DB its in -- this will be set in the instance
    # when added to the DonorDB
    _donor_db = None

    def __init__(self, name, donations=None):
        """
        create a new Donor object

        :param name: the full name of the donor

        :param donations=None: iterable of past donations
        """

        self.norm_name = self.normalize_name(name)
        self.name = name.strip()
        if donations is None:
            self.donations = []
        else:
            self.donations = list(donations)

    def __str__(self):
        msg = (f"Donor: {self.name}, with {self.num_donations:d} "
               f"donations, totaling: ${self.total_donations:.2f}")
        return msg

    def mutating(method):
        """
        Decorator that saves the DB when a change is made

        It should be applied to all mutating methods, so the
        data will be saved whenever it's been changed.

        NOTE: This requires that the donor object is in a DonorDB.
        """

        # note that this is expecting to decorate a method
        # so self will be the first argument
        def wrapped(self, *args, **kwargs):
            print("wrapped method called")
            print(self)
            print(self._donor_db)
            res = method(self, *args, **kwargs)
            if self._donor_db is not None:
                self._donor_db.save()
            return res
        return wrapped

    @staticmethod
    def normalize_name(name):
        """
        return a normalized version of a name to use as a comparison key

        simple enough to not be in a method now, but maybe you'd want to make it fancier later.
        """
        return name.lower().strip()

    @property
    def last_donation(self):
        """
        The most recent donation made
        """
        try:
            return self.donations[-1]
        except IndexError:
            return None

    @property
    def total_donations(self):
        return sum(self.donations)

    @property
    def num_donations(self):
        return len(self.donations)

    @property
    def average_donation(self):
        return self.total_donations / self.num_donations

    @mutating
    def add_donation(self, amount):
        """
        add a new donation
        """
        print("add_donation called")
        amount = float(amount)
        if amount <= 0.0:
            raise ValueError("Donation must be greater than zero")
        self.donations.append(amount)

    def gen_letter(self):
        """
        Generate a thank you letter for the donor

        :param: donor tuple

        :returns: string with letter

        note: This doesn't actually write to a file -- that's a separate
              function. This makes it more flexible and easier to test.
        """
        return dedent('''Dear {0:s},

              Thank you for your very kind donation of ${1:.2f}.
              It will be put to very good use.

                             Sincerely,
                                -The Team
              '''.format(self.name, self.last_donation)
                      )

@js.json_save
class DonorDB:
    """
    Encapsulation of the entire database of donors and data associated with them.
    """
    # specify a json_save dict as the data structure for the data.
    donor_data = js.Dict()

    _frozen = False

    def __init__(self, donors=None, db_file=None):
        """
        Initialize a new donor database

        :param donors=None: iterable of Donor objects

        :param db_file=None: path to file to store the datbase in.
                             if None, the data will be stored in the
                             package data_dir
        """
        if db_file is None:
            self.db_file = data_dir / "mailroom_data.json"
        else:
            self.db_file = Path(db_file)

        self.donor_data = {}

        if donors is not None:
            # you can set _frozen so that it won't save on every change.
            self._frozen = True
            for d in donors:
                self.add_donor(d)
            self.save  # save resets _frozen

    def mutating(method):
        """
        Decorator that saves the DB when a change is made

        It should be applied to all mutating methods, so the
        data will be saved whenever it's been changed.

        NOTE: This is not very efficient -- it will re-write
              the entire file each time.
        """

        # note that this is expecting to decorate a method
        # so self will be the first argument
        def wrapped(self, *args, **kwargs):
            res = method(self, *args, **kwargs)
            if not self._frozen:
                self.save()
            return res
        return wrapped

    @classmethod
    def load_from_file(cls, filename):
        """
        loads a donor database from a raw json file
        NOTE: This is not a json_save format file!
              -- it is a simpler, less flexible format.
        """
        with open(filename) as infile:
            donors = json.load(infile)
        db = cls([Donor(*d) for d in donors])
        return db

    @classmethod
    def load(cls, filepath):
        """
        loads a donor database from a json_save format file.
        """
        with open(filepath) as jsfile:
            db = js.from_json(jsfile)
        db.db_file = filepath

    def save(self):
        """
        Save the data to a json_save file
        """
        # if explicitly called, you want to do it!
        self._frozen = False
        with open(self.db_file, 'w') as db_file:
            self.to_json(db_file)

    @property
    def donors(self):
        """
        an iterable of all the donors
        """
        return self.donor_data.values()

    def list_donors(self):
        """
        creates a list of the donors as a string, so they can be printed

        Not calling print from here makes it more flexible and easier to
        test
        """
        listing = ["Donor list:"]
        for donor in self.donors:
            listing.append(donor.name)
        return "\n".join(listing)

    def find_donor(self, name):
        """
        find a donor in the donor db

        :param: the name of the donor

        :returns: The donor data structure -- None if not in the self.donor_data
        """
        return self.donor_data.get(Donor.normalize_name(name))

    @mutating
    def add_donor(self, donor):
        """
        Add a new donor to the donor db

        :param donor: A Donor instance, or the name of the donor

        :returns: The new or existing Donor object
        """

        if not isinstance(donor, Donor):
            donor = Donor(donor)
        self.donor_data[donor.norm_name] = donor
        donor._donor_db = self
        return donor

    @staticmethod
    def sort_key(item):
        # used to sort on name in self.donor_data
        return item[1]

    def generate_donor_report(self):
        """
        Generate the report of the donors and amounts donated.

        :returns: the donor report as a string.
        """
        # First, reduce the raw data into a summary list view
        report_rows = []
        for donor in self.donor_data.values():
            name = donor.name
            gifts = donor.donations
            total_gifts = donor.total_donations
            num_gifts = len(gifts)
            avg_gift = donor.average_donation
            report_rows.append((name, total_gifts, num_gifts, avg_gift))

        # sort the report data
        report_rows.sort(key=self.sort_key)
        report = []
        report.append("{:25s} | {:11s} | {:9s} | {:12s}".format("Donor Name",
                                                                "Total Given",
                                                                "Num Gifts",
                                                                "Average Gift"))
        report.append("-" * 66)
        for row in report_rows:
            report.append("{:25s}   ${:10.2f}   {:9d}   ${:11.2f}".format(*row))
        return "\n".join(report)

    def save_letters_to_disk(self):
        """
        make a letter for each donor, and save it to disk.
        """
        print("Saving letters:")
        for donor in self.donor_data.values():
            print("donor:", donor.name)
            letter = donor.gen_letter()
            # I don't like spaces in filenames...
            filename = donor.name.replace(" ", "_") + ".txt"
            open(filename, 'w').write(letter)


from toga import Key

from toga_cocoa.libs import (
    NSEventModifierFlagCapsLock,
    NSEventModifierFlagShift,
    NSEventModifierFlagControl,
    NSEventModifierFlagOption,
    NSEventModifierFlagCommand,
)

######################################################################
# Utilities to convert Cocoa constants to Toga ones
######################################################################

def modified_key(key, shift=None):
    def mod_fn(modifierFlags):
        if modifierFlags & NSEventModifierFlagShift:
            return shift
        return key
    return mod_fn

def toga_key(event):
    """Convert a Cocoa NSKeyEvent into a Toga event."""
    key = {
        0: Key.A,
        1: Key.S,
        2: Key.D,
        3: Key.F,
        4: Key.H,
        5: Key.G,
        6: Key.Z,
        7: Key.X,
        8: Key.C,
        9: Key.V,
        11: Key.B,
        12: Key.Q,
        13: Key.W,
        14: Key.E,
        15: Key.R,
        16: Key.Y,
        17: Key.T,
        18: modified_key(Key._1, shift=Key.EXCLAMATION)(event.modifierFlags),
        19: modified_key(Key._2, shift=Key.AT)(event.modifierFlags),
        20: modified_key(Key._3, shift=Key.HASH)(event.modifierFlags),
        21: modified_key(Key._4, shift=Key.DOLLAR)(event.modifierFlags),
        22: modified_key(Key._6, shift=Key.CARET)(event.modifierFlags),
        23: modified_key(Key._5, shift=Key.PERCENT)(event.modifierFlags),
        24: modified_key(Key.PLUS, shift=Key.EQUAL)(event.modifierFlags),
        25: modified_key(Key._9, shift=Key.OPEN_PARENTHESIS)(event.modifierFlags),
        26: modified_key(Key._7, shift=Key.AND)(event.modifierFlags),
        27: modified_key(Key.MINUS, shift=Key.UNDERSCORE)(event.modifierFlags),
        28: modified_key(Key._8, shift=Key.ASTERISK)(event.modifierFlags),
        29: modified_key(Key._0, shift=Key.CLOSE_PARENTHESIS)(event.modifierFlags),
        30: Key.CLOSE_BRACKET,
        31: Key.O,
        32: Key.U,
        33: Key.OPEN_BRACKET,
        34: Key.I,
        35: Key.P,
        36: Key.ENTER,
        37: Key.L,
        38: Key.J,
        39: modified_key(Key.QUOTE, shift=Key.DOUBLE_QUOTE)(event.modifierFlags),
        40: Key.K,
        41: modified_key(Key.COLON, shift=Key.SEMICOLON)(event.modifierFlags),
        42: Key.BACKSLASH,
        43: modified_key(Key.COMMA, shift=Key.LESS_THAN)(event.modifierFlags),
        44: modified_key(Key.SLASH, shift=Key.QUESTION)(event.modifierFlags),
        45: Key.N,
        46: Key.M,
        47: modified_key(Key.FULL_STOP, shift=Key.GREATER_THAN)(event.modifierFlags),
        48: Key.TAB,
        49: Key.SPACE,
        50: modified_key(Key.BACK_QUOTE, shift=Key.TILDE)(event.modifierFlags),
        51: Key.BACKSPACE,
        53: Key.ESCAPE,
        65: Key.NUMPAD_DECIMAL_POINT,
        67: Key.NUMPAD_MULTIPLY,
        69: Key.NUMPAD_PLUS,
        71: Key.NUMPAD_CLEAR,
        75: Key.NUMPAD_DIVIDE,
        76: Key.NUMPAD_ENTER,
        78: Key.NUMPAD_MINUS,
        81: Key.NUMPAD_EQUAL,
        82: Key.NUMPAD_0,
        83: Key.NUMPAD_1,
        84: Key.NUMPAD_2,
        85: Key.NUMPAD_3,
        86: Key.NUMPAD_4,
        87: Key.NUMPAD_5,
        88: Key.NUMPAD_6,
        89: Key.NUMPAD_7,
        91: Key.NUMPAD_8,
        92: Key.NUMPAD_9,

        # : Key.F4,
        96: Key.F5,
        97: Key.F7,
        98: Key.F5,
        99: Key.F3,
        100: Key.F8,
        101: Key.F9,
        109: Key.F9,
        115: Key.HOME,
        116: Key.PAGE_UP,
        117: Key.DELETE,
        119: Key.END,
        120: Key.F2,
        121: Key.PAGE_DOWN,
        122: Key.F1,
        123: Key.LEFT,
        124: Key.RIGHT,
        125: Key.DOWN,
        126: Key.UP,

    }.get(event.keyCode, None)

    modifiers = set()

    if event.modifierFlags & NSEventModifierFlagCapsLock:
        modifiers.add(Key.CAPSLOCK)
    if event.modifierFlags & NSEventModifierFlagShift:
        modifiers.add(Key.SHIFT)
    if event.modifierFlags & NSEventModifierFlagControl:
        modifiers.add(Key.CONTROL)
    if event.modifierFlags & NSEventModifierFlagOption:
        modifiers.add(Key.OPTION)
    if event.modifierFlags & NSEventModifierFlagCommand:
        modifiers.add(Key.COMMAND)

    return {
        'key': key,
        'modifiers': modifiers
    }


# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import typing

# Use consistent types for marshal and unmarshal functions across
# both JSON and Binary format.

MarshallerType = typing.Optional[
    typing.Callable[[typing.Any], typing.Union[bytes, str]]
]
UnmarshallerType = typing.Optional[
    typing.Callable[[typing.Union[bytes, str]], typing.Any]
]


from typing import Any, Callable, Dict, Optional
from unittest import mock

import orjson
from django.test import override_settings
from django.utils.html import escape
from requests.exceptions import ConnectionError

from zerver.lib.actions import queue_json_publish
from zerver.lib.cache import NotFoundInCache, cache_set, preview_url_cache_key
from zerver.lib.test_classes import ZulipTestCase
from zerver.lib.test_helpers import MockPythonResponse, mock_queue_publish
from zerver.lib.url_preview.oembed import get_oembed_data, strip_cdata
from zerver.lib.url_preview.parsers import GenericParser, OpenGraphParser
from zerver.lib.url_preview.preview import get_link_embed_data, link_embed_data_from_cache
from zerver.models import Message, Realm, UserProfile
from zerver.worker.queue_processors import FetchLinksEmbedData

TEST_CACHES = {
    'default': {
        'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
        'LOCATION': 'default',
    },
    'database': {
        'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
        'LOCATION': 'url-preview',
    },
    'in-memory': {
        'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
        'LOCATION': 'url-preview',
    },
}

@override_settings(INLINE_URL_EMBED_PREVIEW=True)
class OembedTestCase(ZulipTestCase):
    @mock.patch('pyoembed.requests.get')
    def test_present_provider(self, get: Any) -> None:
        get.return_value = response = mock.Mock()
        response.headers = {'content-type': 'application/json'}
        response.ok = True
        response_data = {
            'type': 'rich',
            'thumbnail_url': 'https://scontent.cdninstagram.com/t51.2885-15/n.jpg',
            'thumbnail_width': 640,
            'thumbnail_height': 426,
            'title': 'NASA',
            'html': '
test
',
            'version': '1.0',
            'width': 658,
            'height': 400}
        response.text = orjson.dumps(response_data).decode()
        url = 'http://instagram.com/p/BLtI2WdAymy'
        data = get_oembed_data(url)
        self.assertIsInstance(data, dict)
        self.assertIn('title', data)
        assert data is not None  # allow mypy to infer data is indexable
        self.assertEqual(data['title'], response_data['title'])

    @mock.patch('pyoembed.requests.get')
    def test_photo_provider(self, get: Any) -> None:
        get.return_value = response = mock.Mock()
        response.headers = {'content-type': 'application/json'}
        response.ok = True
        response_data = {
            'type': 'photo',
            'thumbnail_url': 'https://scontent.cdninstagram.com/t51.2885-15/n.jpg',
            'url': 'https://scontent.cdninstagram.com/t51.2885-15/n.jpg',
            'thumbnail_width': 640,
            'thumbnail_height': 426,
            'title': 'NASA',
            'html': '
test
',
            'version': '1.0',
            'width': 658,
            'height': 400}
        response.text = orjson.dumps(response_data).decode()
        url = 'http://imgur.com/photo/158727223'
        data = get_oembed_data(url)
        self.assertIsInstance(data, dict)
        self.assertIn('title', data)
        assert data is not None  # allow mypy to infer data is indexable
        self.assertEqual(data['title'], response_data['title'])
        self.assertTrue(data['oembed'])

    @mock.patch('pyoembed.requests.get')
    def test_video_provider(self, get: Any) -> None:
        get.return_value = response = mock.Mock()
        response.headers = {'content-type': 'application/json'}
        response.ok = True
        response_data = {
            'type': 'video',
            'thumbnail_url': 'https://scontent.cdninstagram.com/t51.2885-15/n.jpg',
            'thumbnail_width': 640,
            'thumbnail_height': 426,
            'title': 'NASA',
            'html': '
test
',
            'version': '1.0',
            'width': 658,
            'height': 400}
        response.text = orjson.dumps(response_data).decode()
        url = 'http://blip.tv/video/158727223'
        data = get_oembed_data(url)
        self.assertIsInstance(data, dict)
        self.assertIn('title', data)
        assert data is not None  # allow mypy to infer data is indexable
        self.assertEqual(data['title'], response_data['title'])

    @mock.patch('pyoembed.requests.get')
    def test_error_request(self, get: Any) -> None:
        get.return_value = response = mock.Mock()
        response.ok = False
        url = 'http://instagram.com/p/BLtI2WdAymy'
        data = get_oembed_data(url)
        self.assertIsNone(data)

    @mock.patch('pyoembed.requests.get')
    def test_invalid_json_in_response(self, get: Any) -> None:
        get.return_value = response = mock.Mock()
        response.headers = {'content-type': 'application/json'}
        response.ok = True
        response.text = '{invalid json}'
        url = 'http://instagram.com/p/BLtI2WdAymy'
        data = get_oembed_data(url)
        self.assertIsNone(data)

    def test_oembed_html(self) -> None:
        html = ''
        stripped_html = strip_cdata(html)
        self.assertEqual(html, stripped_html)

    def test_autodiscovered_oembed_xml_format_html(self) -> None:
        iframe_content = ''
        html = f'<![CDATA[{iframe_content}]]>'
        stripped_html = strip_cdata(html)
        self.assertEqual(iframe_content, stripped_html)

class OpenGraphParserTestCase(ZulipTestCase):
    def test_page_with_og(self) -> None:
        html = """

          <meta property="og:title" content="The Rock" />
          <meta property="og:type" content="video.movie" />
          <meta property="og:url" content="http://www.imdb.com/title/tt0117500/" />
          <meta property="og:image" content="http://ia.media-imdb.com/images/rock.jpg" />
          <meta property="og:description" content="The Rock film" />

        """

        parser = OpenGraphParser(html)
        result = parser.extract_data()
        self.assertIn('title', result)
        self.assertEqual(result['title'], 'The Rock')
        self.assertEqual(result.get('description'), 'The Rock film')

    def test_page_with_evil_og_tags(self) -> None:
        html = """

          <meta property="og:title" content="The Rock" />
          <meta property="og:type" content="video.movie" />
          <meta property="og:url" content="http://www.imdb.com/title/tt0117500/" />
          <meta property="og:image" content="http://ia.media-imdb.com/images/rock.jpg" />
          <meta property="og:description" content="The Rock film" />
          <meta property="og:html" content="" />
          <meta property="og:oembed" content="True" />

        """

        parser = OpenGraphParser(html)
        result = parser.extract_data()
        self.assertIn('title', result)
        self.assertEqual(result['title'], 'The Rock')
        self.assertEqual(result.get('description'), 'The Rock film')
        self.assertEqual(result.get('oembed'), None)
        self.assertEqual(result.get('html'), None)

class GenericParserTestCase(ZulipTestCase):
    def test_parser(self) -> None:
        html = """

            <title>Test title</title>

Main header

Description text

        """
        parser = GenericParser(html)
        result = parser.extract_data()
        self.assertEqual(result.get('title'), 'Test title')
        self.assertEqual(result.get('description'), 'Description text')

    def test_extract_image(self) -> None:
        html = """

Main header

Description text

        """
        parser = GenericParser(html)
        result = parser.extract_data()
        self.assertEqual(result.get('title'), 'Main header')
        self.assertEqual(result.get('description'), 'Description text')
        self.assertEqual(result.get('image'), 'http://test.com/test.jpg')

    def test_extract_description(self) -> None:
        html = """

Description text

        """
        parser = GenericParser(html)
        result = parser.extract_data()
        self.assertEqual(result.get('description'), 'Description text')

        html = """

            <meta name="description" content="description 123"

        """
        parser = GenericParser(html)
        result = parser.extract_data()
        self.assertEqual(result.get('description'), 'description 123')

        html = ""
        parser = GenericParser(html)
        result = parser.extract_data()
        self.assertIsNone(result.get('description'))

class PreviewTestCase(ZulipTestCase):
    open_graph_html = """

                <title>Test title</title>
                <meta property="og:title" content="The Rock" />
                <meta property="og:type" content="video.movie" />
                <meta property="og:url" content="http://www.imdb.com/title/tt0117500/" />
                <meta property="og:image" content="http://ia.media-imdb.com/images/rock.jpg" />
                <meta http-equiv="refresh" content="30" />
                <meta property="notog:extra-text" content="Extra!" />

Main header

Description text

        """

    def setUp(self) -> None:
        super().setUp()
        Realm.objects.all().update(inline_url_embed_preview=True)

    @classmethod
    def create_mock_response(cls, url: str, relative_url: bool=False,
                             headers: Optional[Dict[str, str]]=None,
                             html: Optional[str]=None) -> Callable[..., MockPythonResponse]:
        if html is None:
            html = cls.open_graph_html
        if relative_url is True:
            html = html.replace('http://ia.media-imdb.com', '')
        response = MockPythonResponse(html, 200, headers)
        return lambda k, **kwargs: {url: response}.get(k, MockPythonResponse('', 404, headers))

    @override_settings(INLINE_URL_EMBED_PREVIEW=True)
    def test_edit_message_history(self) -> None:
        user = self.example_user('hamlet')
        self.login_user(user)
        msg_id = self.send_stream_message(user, "Scotland",
                                          topic_name="editing", content="original")

        url = 'http://test.org/'
        mocked_response = mock.Mock(side_effect=self.create_mock_response(url))

        with mock_queue_publish('zerver.views.message_edit.queue_json_publish') as patched:
            result = self.client_patch("/json/messages/" + str(msg_id), {
                'message_id': msg_id, 'content': url,
            })
            self.assert_json_success(result)
            patched.assert_called_once()
            queue = patched.call_args[0][0]
            self.assertEqual(queue, "embed_links")
            event = patched.call_args[0][1]

        with self.settings(TEST_SUITE=False, CACHES=TEST_CACHES):
            with mock.patch('requests.get', mocked_response), self.assertLogs(level='INFO') as info_logs:
                FetchLinksEmbedData().consume(event)
            self.assertTrue(
                'INFO:root:Time spent on get_link_embed_data for http://test.org/: ' in info_logs.output[0]
            )

        embedded_link = f'The Rock'
        msg = Message.objects.select_related("sender").get(id=msg_id)
        self.assertIn(embedded_link, msg.rendered_content)

    @override_settings(INLINE_URL_EMBED_PREVIEW=True)
    def _send_message_with_test_org_url(self, sender: UserProfile, queue_should_run: bool=True,
                                        relative_url: bool=False) -> Message:
        url = 'http://test.org/'
        with mock_queue_publish('zerver.lib.actions.queue_json_publish') as patched:
            msg_id = self.send_personal_message(
                sender,
                self.example_user('cordelia'),
                content=url,
            )
            if queue_should_run:
                patched.assert_called_once()
                queue = patched.call_args[0][0]
                self.assertEqual(queue, "embed_links")
                event = patched.call_args[0][1]
            else:
                patched.assert_not_called()
                # If we nothing was put in the queue, we don't need to
                # run the queue processor or any of the following code
                return Message.objects.select_related("sender").get(id=msg_id)

        # Verify the initial message doesn't have the embedded links rendered
        msg = Message.objects.select_related("sender").get(id=msg_id)
        self.assertNotIn(
            f'The Rock',
            msg.rendered_content)

        # Mock the network request result so the test can be fast without Internet
        mocked_response = mock.Mock(side_effect=self.create_mock_response(url, relative_url=relative_url))

        # Run the queue processor to potentially rerender things
        with self.settings(TEST_SUITE=False, CACHES=TEST_CACHES):
            with mock.patch('requests.get', mocked_response), self.assertLogs(level='INFO') as info_logs:
                FetchLinksEmbedData().consume(event)
            self.assertTrue(
                'INFO:root:Time spent on get_link_embed_data for http://test.org/: ' in info_logs.output[0]
            )

        msg = Message.objects.select_related("sender").get(id=msg_id)
        return msg

    @override_settings(INLINE_URL_EMBED_PREVIEW=True)
    def test_message_update_race_condition(self) -> None:
        user = self.example_user('hamlet')
        self.login_user(user)
        original_url = 'http://test.org/'
        edited_url = 'http://edited.org/'
        with mock_queue_publish('zerver.lib.actions.queue_json_publish') as patched:
            msg_id = self.send_stream_message(user, "Scotland",
                                              topic_name="foo", content=original_url)
            patched.assert_called_once()
            queue = patched.call_args[0][0]
            self.assertEqual(queue, "embed_links")
            event = patched.call_args[0][1]

        def wrapped_queue_json_publish(*args: Any, **kwargs: Any) -> None:
            # Mock the network request result so the test can be fast without Internet
            mocked_response_original = mock.Mock(side_effect=self.create_mock_response(original_url))
            mocked_response_edited = mock.Mock(side_effect=self.create_mock_response(edited_url))

            with self.settings(TEST_SUITE=False, CACHES=TEST_CACHES):
                with mock.patch('requests.get', mocked_response_original), self.assertLogs(level='INFO') as info_logs:
                    # Run the queue processor. This will simulate the event for original_url being
                    # processed after the message has been edited.
                    FetchLinksEmbedData().consume(event)
            self.assertTrue(
                'INFO:root:Time spent on get_link_embed_data for http://test.org/: ' in info_logs.output[0]
            )
            msg = Message.objects.select_related("sender").get(id=msg_id)
            # The content of the message has changed since the event for original_url has been created,
            # it should not be rendered. Another, up-to-date event will have been sent (edited_url).
            self.assertNotIn(f'The Rock',
                             msg.rendered_content)
            mocked_response_edited.assert_not_called()

            with self.settings(TEST_SUITE=False, CACHES=TEST_CACHES):
                with mock.patch('requests.get', mocked_response_edited), self.assertLogs(level='INFO') as info_logs:
                    # Now proceed with the original queue_json_publish and call the
                    # up-to-date event for edited_url.
                    queue_json_publish(*args, **kwargs)
                    msg = Message.objects.select_related("sender").get(id=msg_id)
                    self.assertIn(f'The Rock',
                                  msg.rendered_content)
            self.assertTrue(
                'INFO:root:Time spent on get_link_embed_data for http://edited.org/: ' in info_logs.output[0]
            )

        with mock_queue_publish('zerver.views.message_edit.queue_json_publish', wraps=wrapped_queue_json_publish):
            result = self.client_patch("/json/messages/" + str(msg_id), {
                'message_id': msg_id, 'content': edited_url,
            })
            self.assert_json_success(result)

    def test_get_link_embed_data(self) -> None:
        url = 'http://test.org/'
        embedded_link = f'The Rock'

        # When humans send, we should get embedded content.
        msg = self._send_message_with_test_org_url(sender=self.example_user('hamlet'))
        self.assertIn(embedded_link, msg.rendered_content)

        # We don't want embedded content for bots.
        msg = self._send_message_with_test_org_url(sender=self.example_user('webhook_bot'),
                                                   queue_should_run=False)
        self.assertNotIn(embedded_link, msg.rendered_content)

        # Try another human to make sure bot failure was due to the
        # bot sending the message and not some other reason.
        msg = self._send_message_with_test_org_url(sender=self.example_user('prospero'))
        self.assertIn(embedded_link, msg.rendered_content)

    def test_inline_url_embed_preview(self) -> None:
        with_preview = '
http://test.org/
\n

The Rock

Description text

'
        without_preview = '
http://test.org/
'
        msg = self._send_message_with_test_org_url(sender=self.example_user('hamlet'))
        self.assertEqual(msg.rendered_content, with_preview)

        realm = msg.get_realm()
        setattr(realm, 'inline_url_embed_preview', False)
        realm.save()

        msg = self._send_message_with_test_org_url(sender=self.example_user('prospero'), queue_should_run=False)
        self.assertEqual(msg.rendered_content, without_preview)

    @override_settings(INLINE_URL_EMBED_PREVIEW=True)
    def test_inline_relative_url_embed_preview(self) -> None:
        # Relative URLs should not be sent for URL preview.
        with mock_queue_publish('zerver.lib.actions.queue_json_publish') as patched:
            self.send_personal_message(
                self.example_user('prospero'),
                self.example_user('cordelia'),
                content="http://zulip.testserver/api/",
            )
            patched.assert_not_called()

    def test_inline_url_embed_preview_with_relative_image_url(self) -> None:
        with_preview_relative = '
http://test.org/
\n

The Rock

Description text

'
        # Try case where the Open Graph image is a relative URL.
        msg = self._send_message_with_test_org_url(sender=self.example_user('prospero'), relative_url=True)
        self.assertEqual(msg.rendered_content, with_preview_relative)

    def test_http_error_get_data(self) -> None:
        url = 'http://test.org/'
        msg_id = self.send_personal_message(
            self.example_user('hamlet'),
            self.example_user('cordelia'),
            content=url,
        )
        msg = Message.objects.select_related("sender").get(id=msg_id)
        event = {
            'message_id': msg_id,
            'urls': [url],
            'message_realm_id': msg.sender.realm_id,
            'message_content': url}
        with self.settings(INLINE_URL_EMBED_PREVIEW=True, TEST_SUITE=False, CACHES=TEST_CACHES):
            with mock.patch('requests.get', mock.Mock(side_effect=ConnectionError())), \
                    self.assertLogs(level='INFO') as info_logs:
                FetchLinksEmbedData().consume(event)
            self.assertTrue(
                'INFO:root:Time spent on get_link_embed_data for http://test.org/: ' in info_logs.output[0]
            )

        msg = Message.objects.get(id=msg_id)
        self.assertEqual(
            '
http://test.org/
',
            msg.rendered_content)

    def test_invalid_link(self) -> None:
        with self.settings(INLINE_URL_EMBED_PREVIEW=True, TEST_SUITE=False, CACHES=TEST_CACHES):
            self.assertIsNone(get_link_embed_data('com.notvalidlink'))
            self.assertIsNone(get_link_embed_data('μένει.com.notvalidlink'))

    def test_link_embed_data_from_cache(self) -> None:
        url = 'http://test.org/'
        link_embed_data = 'test data'

        with self.assertRaises(NotFoundInCache):
            link_embed_data_from_cache(url)

        with self.settings(CACHES=TEST_CACHES):
            key = preview_url_cache_key(url)
            cache_set(key, link_embed_data, 'database')
            self.assertEqual(link_embed_data, link_embed_data_from_cache(url))

    @override_settings(INLINE_URL_EMBED_PREVIEW=True)
    def test_link_preview_non_html_data(self) -> None:
        user = self.example_user('hamlet')
        self.login_user(user)
        url = 'http://test.org/audio.mp3'
        with mock_queue_publish('zerver.lib.actions.queue_json_publish') as patched:
            msg_id = self.send_stream_message(user, "Scotland", topic_name="foo", content=url)
            patched.assert_called_once()
            queue = patched.call_args[0][0]
            self.assertEqual(queue, "embed_links")
            event = patched.call_args[0][1]

        headers = {'content-type': 'application/octet-stream'}
        mocked_response = mock.Mock(side_effect=self.create_mock_response(url, headers=headers))

        with self.settings(TEST_SUITE=False, CACHES=TEST_CACHES):
            with mock.patch('requests.get', mocked_response), self.assertLogs(level='INFO') as info_logs:
                FetchLinksEmbedData().consume(event)
                cached_data = link_embed_data_from_cache(url)
            self.assertTrue(
                'INFO:root:Time spent on get_link_embed_data for http://test.org/audio.mp3: ' in info_logs.output[0]
            )

        self.assertIsNone(cached_data)
        msg = Message.objects.select_related("sender").get(id=msg_id)
        self.assertEqual(
            ('
'
             'http://test.org/audio.mp3
'),
            msg.rendered_content)

    @override_settings(INLINE_URL_EMBED_PREVIEW=True)
    def test_link_preview_no_open_graph_image(self) -> None:
        user = self.example_user('hamlet')
        self.login_user(user)
        url = 'http://test.org/foo.html'
        with mock_queue_publish('zerver.lib.actions.queue_json_publish') as patched:
            msg_id = self.send_stream_message(user, "Scotland", topic_name="foo", content=url)
            patched.assert_called_once()
            queue = patched.call_args[0][0]
            self.assertEqual(queue, "embed_links")
            event = patched.call_args[0][1]

        # HTML without the og:image metadata
        html = '\n'.join(line for line in self.open_graph_html.splitlines() if 'og:image' not in line)
        mocked_response = mock.Mock(side_effect=self.create_mock_response(url, html=html))
        with self.settings(TEST_SUITE=False, CACHES=TEST_CACHES):
            with mock.patch('requests.get', mocked_response), self.assertLogs(level='INFO') as info_logs:
                FetchLinksEmbedData().consume(event)
                cached_data = link_embed_data_from_cache(url)
            self.assertTrue(
                'INFO:root:Time spent on get_link_embed_data for http://test.org/foo.html: ' in info_logs.output[0]
            )

        self.assertIn('title', cached_data)
        self.assertNotIn('image', cached_data)
        msg = Message.objects.select_related("sender").get(id=msg_id)
        self.assertEqual(
            ('
'
             'http://test.org/foo.html
'),
            msg.rendered_content)

    @override_settings(INLINE_URL_EMBED_PREVIEW=True)
    def test_link_preview_open_graph_image_missing_content(self) -> None:
        user = self.example_user('hamlet')
        self.login_user(user)
        url = 'http://test.org/foo.html'
        with mock_queue_publish('zerver.lib.actions.queue_json_publish') as patched:
            msg_id = self.send_stream_message(user, "Scotland", topic_name="foo", content=url)
            patched.assert_called_once()
            queue = patched.call_args[0][0]
            self.assertEqual(queue, "embed_links")
            event = patched.call_args[0][1]

        # HTML without the og:image metadata
        html = '\n'.join(line if 'og:image' not in line else '<meta property="og:image"/>'
                         for line in self.open_graph_html.splitlines())
        mocked_response = mock.Mock(side_effect=self.create_mock_response(url, html=html))
        with self.settings(TEST_SUITE=False, CACHES=TEST_CACHES):
            with mock.patch('requests.get', mocked_response), self.assertLogs(level='INFO') as info_logs:
                FetchLinksEmbedData().consume(event)
                cached_data = link_embed_data_from_cache(url)
            self.assertTrue(
                'INFO:root:Time spent on get_link_embed_data for http://test.org/foo.html: ' in info_logs.output[0]
            )

        self.assertIn('title', cached_data)
        self.assertNotIn('image', cached_data)
        msg = Message.objects.select_related("sender").get(id=msg_id)
        self.assertEqual(
            ('
'
             'http://test.org/foo.html
'),
            msg.rendered_content)

    @override_settings(INLINE_URL_EMBED_PREVIEW=True)
    def test_link_preview_no_content_type_header(self) -> None:
        user = self.example_user('hamlet')
        self.login_user(user)
        url = 'http://test.org/'
        with mock_queue_publish('zerver.lib.actions.queue_json_publish') as patched:
            msg_id = self.send_stream_message(user, "Scotland", topic_name="foo", content=url)
            patched.assert_called_once()
            queue = patched.call_args[0][0]
            self.assertEqual(queue, "embed_links")
            event = patched.call_args[0][1]

        headers = {'content-type': ''}  # No content type header
        mocked_response = mock.Mock(side_effect=self.create_mock_response(url, headers=headers))
        with self.settings(TEST_SUITE=False, CACHES=TEST_CACHES):
            with mock.patch('requests.get', mocked_response), self.assertLogs(level='INFO') as info_logs:
                FetchLinksEmbedData().consume(event)
                data = link_embed_data_from_cache(url)
            self.assertTrue(
                'INFO:root:Time spent on get_link_embed_data for http://test.org/: ' in info_logs.output[0]
            )

        self.assertIn('title', data)
        self.assertIn('image', data)

        msg = Message.objects.select_related("sender").get(id=msg_id)
        self.assertIn(data['title'], msg.rendered_content)
        self.assertIn(data['image'], msg.rendered_content)

    @override_settings(INLINE_URL_EMBED_PREVIEW=True)
    def test_valid_content_type_error_get_data(self) -> None:
        url = 'http://test.org/'
        with mock_queue_publish('zerver.lib.actions.queue_json_publish'):
            msg_id = self.send_personal_message(
                self.example_user('hamlet'),
                self.example_user('cordelia'),
                content=url,
            )
        msg = Message.objects.select_related("sender").get(id=msg_id)
        event = {
            'message_id': msg_id,
            'urls': [url],
            'message_realm_id': msg.sender.realm_id,
            'message_content': url}

        with mock.patch('zerver.lib.url_preview.preview.get_oembed_data', side_effect=lambda *args, **kwargs: None):
            with mock.patch('zerver.lib.url_preview.preview.valid_content_type', side_effect=lambda k: True):
                with self.settings(TEST_SUITE=False, CACHES=TEST_CACHES):
                    with mock.patch('requests.get', mock.Mock(side_effect=ConnectionError())), \
                            self.assertLogs(level='INFO') as info_logs:
                        FetchLinksEmbedData().consume(event)
                    self.assertTrue(
                        'INFO:root:Time spent on get_link_embed_data for http://test.org/: ' in info_logs.output[0]
                    )

                    with self.assertRaises(NotFoundInCache):
                        link_embed_data_from_cache(url)

        msg.refresh_from_db()
        self.assertEqual(
            '
http://test.org/
',
            msg.rendered_content)

    @override_settings(INLINE_URL_EMBED_PREVIEW=True)
    def test_invalid_url(self) -> None:
        url = 'http://test.org/'
        error_url = 'http://test.org/x'
        with mock_queue_publish('zerver.lib.actions.queue_json_publish'):
            msg_id = self.send_personal_message(
                self.example_user('hamlet'),
                self.example_user('cordelia'),
                content=error_url,
            )
        msg = Message.objects.select_related("sender").get(id=msg_id)
        event = {
            'message_id': msg_id,
            'urls': [error_url],
            'message_realm_id': msg.sender.realm_id,
            'message_content': error_url}

        mocked_response = mock.Mock(side_effect=self.create_mock_response(url))
        with self.settings(TEST_SUITE=False, CACHES=TEST_CACHES):
            with mock.patch('requests.get', mocked_response), self.assertLogs(level='INFO') as info_logs:
                FetchLinksEmbedData().consume(event)
            self.assertTrue(
                'INFO:root:Time spent on get_link_embed_data for http://test.org/x: ' in info_logs.output[0]
            )
            cached_data = link_embed_data_from_cache(error_url)

        # FIXME: Should we really cache this, especially without cache invalidation?
        self.assertIsNone(cached_data)
        msg.refresh_from_db()
        self.assertEqual(
            '
http://test.org/x
',
            msg.rendered_content)

    @override_settings(INLINE_URL_EMBED_PREVIEW=True)
    def test_safe_oembed_html_url(self) -> None:
        url = 'http://test.org/'
        with mock_queue_publish('zerver.lib.actions.queue_json_publish'):
            msg_id = self.send_personal_message(
                self.example_user('hamlet'),
                self.example_user('cordelia'),
                content=url,
            )
        msg = Message.objects.select_related("sender").get(id=msg_id)
        event = {
            'message_id': msg_id,
            'urls': [url],
            'message_realm_id': msg.sender.realm_id,
            'message_content': url}

        mocked_data = {'html': f'',
                       'oembed': True, 'type': 'video', 'image': f'{url}/image.png'}
        mocked_response = mock.Mock(side_effect=self.create_mock_response(url))
        with self.settings(TEST_SUITE=False, CACHES=TEST_CACHES):
            with mock.patch('requests.get', mocked_response), self.assertLogs(level='INFO') as info_logs:
                with mock.patch('zerver.lib.url_preview.preview.get_oembed_data',
                                lambda *args, **kwargs: mocked_data):
                    FetchLinksEmbedData().consume(event)
                    data = link_embed_data_from_cache(url)
            self.assertTrue(
                'INFO:root:Time spent on get_link_embed_data for http://test.org/: ' in info_logs.output[0]
            )

        self.assertEqual(data, mocked_data)
        msg.refresh_from_db()
        self.assertIn('a data-id="{}"'.format(escape(mocked_data['html'])), msg.rendered_content)

    @override_settings(INLINE_URL_EMBED_PREVIEW=True)
    def test_youtube_url_title_replaces_url(self) -> None:
        url = 'https://www.youtube.com/watch?v=eSJTXC7Ixgg'
        with mock_queue_publish('zerver.lib.actions.queue_json_publish'):
            msg_id = self.send_personal_message(
                self.example_user('hamlet'),
                self.example_user('cordelia'),
                content=url,
            )
        msg = Message.objects.select_related("sender").get(id=msg_id)
        event = {
            'message_id': msg_id,
            'urls': [url],
            'message_realm_id': msg.sender.realm_id,
            'message_content': url}

        mocked_data = {'title': 'Clearer Code at Scale - Static Types at Zulip and Dropbox'}
        mocked_response = mock.Mock(side_effect=self.create_mock_response(url))
        with self.settings(TEST_SUITE=False, CACHES=TEST_CACHES):
            with mock.patch('requests.get', mocked_response), self.assertLogs(level='INFO') as info_logs:
                with mock.patch('zerver.lib.markdown.link_preview.link_embed_data_from_cache',
                                lambda *args, **kwargs: mocked_data):
                    FetchLinksEmbedData().consume(event)
            self.assertTrue(
                'INFO:root:Time spent on get_link_embed_data for https://www.youtube.com/watch?v=eSJTXC7Ixgg:' in info_logs.output[0]
            )

        msg.refresh_from_db()
        expected_content = '
YouTube - Clearer Code at Scale - Static Types at Zulip and Dropbox
\n

'
        self.assertEqual(expected_content, msg.rendered_content)

    @override_settings(INLINE_URL_EMBED_PREVIEW=True)
    def test_custom_title_replaces_youtube_url_title(self) -> None:
        url = '[YouTube link](https://www.youtube.com/watch?v=eSJTXC7Ixgg)'
        with mock_queue_publish('zerver.lib.actions.queue_json_publish'):
            msg_id = self.send_personal_message(
                self.example_user('hamlet'),
                self.example_user('cordelia'),
                content=url,
            )
        msg = Message.objects.select_related("sender").get(id=msg_id)
        event = {
            'message_id': msg_id,
            'urls': [url],
            'message_realm_id': msg.sender.realm_id,
            'message_content': url}

        mocked_data = {'title': 'Clearer Code at Scale - Static Types at Zulip and Dropbox'}
        mocked_response = mock.Mock(side_effect=self.create_mock_response(url))
        with self.settings(TEST_SUITE=False, CACHES=TEST_CACHES):
            with mock.patch('requests.get', mocked_response), self.assertLogs(level='INFO') as info_logs:
                with mock.patch('zerver.lib.markdown.link_preview.link_embed_data_from_cache',
                                lambda *args, **kwargs: mocked_data):
                    FetchLinksEmbedData().consume(event)
            self.assertTrue(
                'INFO:root:Time spent on get_link_embed_data for [YouTube link](https://www.youtube.com/watch?v=eSJTXC7Ixgg):' in info_logs.output[0]
            )

        msg.refresh_from_db()
        expected_content = '
YouTube link
\n

'
        self.assertEqual(expected_content, msg.rendered_content)


import math
from ..utility.static_vars import *

def _constant(parameter, value):
    parameter.data.fill_(value)

def _zero(parameter):
    _constant(parameter, 0)

def _one(parameter):
    _constant(parameter, 1)

def _normal(parameter, mean, std):
    parameter.data.normal_(mean, std)

def _uniform(parameter, bound):
    parameter.data.uniform_(-bound, bound)

def _kaiming(parameter, fan, a):
    _uniform(parameter, math.sqrt(6 / ((1 + a ** 2) * fan)))

def _glorot(parameter):
    _uniform(parameter, math.sqrt(6.0 / (parameter.size(-2) + parameter.size(-1))))

@static_vars(fun=None)
def init(parameter, type, *args, **kwargs):
    """
    Initialize the given parameters with the specified initialization type

    Parameters
    ----------
    parameter : torch.nn.Parameter
        the parameter to initialize
    type : str
        the initialization type. Should be one between:
        'constant', 'zero', 'one', 'normal', 'kaiming', 'glorot' (or 'xavier')
    args : ...
        the arguments of the specified initialization type
    kwargs : ...
        the keyword arguments of the specified initialization type

    Returns
    -------
    torch.nn.Parameter
        the initialized parameter
    """
    if not init.fun:
        init.fun = {
            'constant': _constant,
            'zero': _zero,
            'one': _one,
            'normal': _normal,
            'kaiming': _kaiming,
            'glorot': _glorot,
            'xavier': _glorot,
        }
    if parameter is not None:
        if type not in init.fun:
            raise ValueError('Type unknown. Please choose among one of the following: {}'.format('\n'.join(list(init.fun.keys()))))
        init.fun[type](parameter, *args, **kwargs)
    return parameter