import bing_face_api as bfa

if __name__ == '__main__':
    '''
    コマンドライン引数を使用する場合
    顔認識する画像のディレクトリ
    search_dir = sys.argv[0]
    '''
    # 顔認識する画像のディレクトリ
    search_dir = "./image/original/"
    # 顔認識する画像のファイル名を取得
    img_path_list = bfa.get_image_path_list(search_dir)
    # 顔認識
    bfa.detect_image(img_path_list)


import pytest

from trackash.users.models import User

pytestmark = pytest.mark.django_db

def test_user_get_absolute_url(user: User):
    assert user.get_absolute_url() == f"/users/{user.username}/"


# -*- coding: utf-8 -*-
# Generated by Django 1.9.4 on 2017-05-16 13:47
from __future__ import unicode_literals

from django.db import migrations

def PopulateLevelCreatedDateField(apps, schema_editor):
    level_model = apps.get_model("indicators", "Level")
    levels = level_model.objects.all()

    for l in levels:
        if l.name == 'Goal':
            l.create_date = '2015-10-03 19:03:50'
        elif l.name == 'Output':
            l.create_date = '2015-10-03 19:03:52'
        elif l.name == 'Outcome':
            l.create_date = '2015-10-03 19:03:51'
        elif l.name == 'Activity':
            l.create_date = '2015-10-03 19:03:53'
        elif l.name == 'Impact':
            l.create_date = '2015-10-03 19:03:54'
        l.save()

class Migration(migrations.Migration):

    dependencies = [
        ('indicators', '0007_auto_20170510_0749'),
    ]

    operations = [
        migrations.RunPython(PopulateLevelCreatedDateField),
    ]


# Licensed under a 3-clause BSD style license - see LICENSE.rst

def get_package_data():
    return {'astropy.nddata.tests': ['data/*.fits']}


# -*- coding: utf-8 -*-

import asyncio
from datetime import datetime

from cmyui import log, Ansi
from cmyui.osu import Mods
from discord import Embed
from discord.ext import commands
from discord.threads import Thread
from tinydb.operations import set as dbset
from tinydb.queries import Query

from objects.sakuro import Sakuro, ContextWrap
from osu.calculator import Calculator
from objects import glob, config
from utils.misc import convert_status_str, convert_str_status, make_safe_name, convert_grade_emoji, sakuru_only
from objects.user import UserHelper
from utils.wrappers import sakuroCommand
from utils.misc import BEATMAP_REGEX

QUEUE_EMOJIS = (
    '1⃣',
    '2⃣',
    '3⃣',
    '4⃣',
    '5⃣'
)

class AdminCog(commands.Cog, name='Admin'):
    """Utilities for admins."""

    def __init__(self, bot: Sakuro):
        self.bot = bot
        self.hide = True

    @sakuroCommand(name='reload', hidden=True)
    @commands.is_owner()
    async def _reload(self, ctx: ContextWrap, module: str):
        """Reloads a module."""
        try:
            self.bot.unload_extension(module)
            self.bot.load_extension(module)
        except Exception as e:
            await ctx.send('\N{PISTOL}')
            await ctx.send('{}: {}'.format(type(e).__name__, e))
        else:
            await ctx.send('\N{OK HAND SIGN}')

    @sakuroCommand(hidden=True)
    @commands.is_owner()
    async def shutdown(self, ctx: ContextWrap) -> None:
        await ctx.send('Night night..')
        await self.bot.close()

    @sakuroCommand(hidden=True)
    @commands.check(sakuru_only)
    @commands.has_permissions(ban_members=True)
    async def replay(self, ctx: ContextWrap, nickname: str, mods: str, map_id: int):
        player = await UserHelper.getOsuUserByName(make_safe_name(nickname), 'info')
        description = ""

        if not player:
            async with glob.http.get("https://sakuru.pw/api/search", params={
                "q": nickname
            }) as resp:
                if resp.status == 200:
                    data = await resp.json()

                    if data['length'] == 0:
                        return await ctx.send(f"Nothing matched with {nickname} not found, check your spelling.")

                    embed = Embed(
                        color=ctx.author.color,
                        timestamp=datetime.now()
                    )

                    embed.set_author(name=f"Search queue for {nickname}")

                    for idx, row in enumerate(data['matches']):
                        description += f"**{idx + 1}.** [{row['name']}](https://sakuru.pw/u/{row['id']})\n"

                    embed.description = description
                    description = ""
                    message = await ctx.send(embed=embed)

                    for emoji in QUEUE_EMOJIS[:data['length']]:
                        await message.add_reaction(emoji)
                    try:
                        reaction, user = await self.bot.wait_for('reaction_add',
                                                                 check=lambda r, u: r.message.id == message.id \
                                                                                and r.emoji in QUEUE_EMOJIS \
                                                                                and u == ctx.author,
                                                                 timeout=60.0)
                    except asyncio.TimeoutError:
                        await ctx.send('Time is out!')
                    else:
                        player = await UserHelper.getOsuUserByName(
                            make_safe_name(
                                data['matches'][QUEUE_EMOJIS.index(reaction.emoji)]['name']),
                            'info'
                        )
                        await message.delete()
                else:
                    return await ctx.send("Error! Try again.")

        scores = await UserHelper.getUserScores(player['id'], 0, mods, 5, 'best', map_id)

        if len(scores) == 0:
            return await ctx.send(f"This player has no scores on `{map_id}`!")

        map_fullname = ""

        for idx, score in enumerate(scores):
            calc = await Calculator.calculate(
                score['beatmap']['id'],
                0,
                score['mods'],
                score['acc'],
                None
            )
            map_fullname = calc['map_fullname']

            description += f"""** {idx + 1}. {f' +{Mods(score["mods"])!r}' if score['mods'] != 0 else ''}** [{calc['stars']:.2f}★]\n""" \
                           f"▸ {convert_grade_emoji(score['grade'])} ▸ **{score['pp']:.2f}PP**" \
                           f"""{f' *({calc["pp"]:.2f}PP for {score["acc"]:.2f}% FC)*' if score['grade'] not in ('S', 'SS', 'X', 'SH') else ''} """ \
                           f"▸ {score['acc']:.2f}%\n▸ {score['score']} ▸ x{score['max_combo']}/{score['beatmap']['max_combo']} " \
                           f"▸ [{score['n300']}/{score['n100']}/{score['n50']}/{score['nmiss']}]\n" \
                           f"▸ [Score Set <t:{datetime.fromisoformat(score['play_time']).timestamp().__int__()}:R>]" \
                           f"(https://osu.sakuru.pw/api/get_replay?id={score['id']})\n"

        embed = Embed(color=ctx.author.color, description=description)

        embed.set_author(name=f"Top {len(scores)} Plays for {player['name']} on {map_fullname}",
                         url=f"https://sakuru.pw/u/{player['id']}",
                         icon_url=f"https://sakuru.pw/static/flags/{player['country'].upper()}.png")
        embed.set_footer(text="Click on Score Set to download replay.",
                         icon_url="https://sakuru.pw/static/ingame.png")

        await ctx.send(embed=embed)

    @sakuroCommand(hidden=True)
    @commands.check(sakuru_only)
    @commands.has_permissions(ban_members=True)
    async def restrict(self, ctx: ContextWrap, nickname: str, *reason: str):
        if not await UserHelper.getOsuUserByName(make_safe_name(nickname), 'info'):
            return await ctx.send(f"Player with nickname {nickname} not found.")

        admin = await UserHelper.getDiscordUser(ctx.message.author.id)

        async with glob.http.get("https://osu.sakuru.pw/api/handle_admin",
                                 params={
                                      "secret": config.API_SECRET,
                                      "action": "restrict",
                                      "nickname": make_safe_name(nickname),
                                      "reason": ' '.join(reason),
                                      "admin": admin['safe_name']
                                  }) as resp:
            if resp.status == 200:
                await ctx.message.add_reaction('\N{OK HAND SIGN}')
            else:
                return await ctx.send("Error occurred.")

    @sakuroCommand(hidden=True)
    @commands.check(sakuru_only)
    @commands.has_permissions(ban_members=True)
    async def unrestrict(self, ctx: ContextWrap, nickname: str, *reason: str):
        if not await UserHelper.getOsuUserByName(make_safe_name(nickname), 'info'):
            return await ctx.send(f"Player with nickname {nickname} not found.")

        admin = await UserHelper.getDiscordUser(ctx.message.author.id)

        async with glob.http.get("https://osu.sakuru.pw/api/handle_admin",
                                 params={
                                      "secret": config.API_SECRET,
                                      "action": "unrestrict",
                                      "nickname": make_safe_name(nickname),
                                      "reason": ' '.join(reason),
                                      "admin": admin['safe_name']
                                  }) as resp:
            if resp.status == 200:
                await ctx.message.add_reaction('\N{OK HAND SIGN}')
            else:
                return await ctx.send("Error occurred.")

    @sakuroCommand(hidden=True)
    @commands.check(sakuru_only)
    @commands.has_permissions(ban_members=True)
    async def rqmap(self, ctx: ContextWrap, status: str, type: str):
        if (
            not isinstance(ctx.message.channel, Thread) or
            not ctx.message.channel.parent_id == config.MAP_REQS
        ):
            return

        if ctx.message.channel.archived:
            return

        req_table = glob.db.table('map_reqs')

        Requests = Query()
        req = req_table.get((Requests.thread_id == ctx.message.channel.id) & (Requests.active == True))

        admin = await UserHelper.getDiscordUser(ctx.message.author.id)

        if not admin:
            return await ctx.send('who are yo')

        if type not in ('map', 'set'):
            msg = await ctx.reply('Invalid type! (map, set)')

            await msg.delete(delay=15)
            await ctx.message.delete(delay=15)
            return

        if status not in ('love', 'rank', 'unrank'):
            msg = await ctx.reply('Invalid status! (love, rank, unrank)')

            await msg.delete(delay=15)
            await ctx.message.delete(delay=15)
            return

        if type == "map":
            params = {
                "set_id": req['beatmap']['set_id']
            }

            async with glob.http.get("https://osu.sakuru.pw/api/get_map_info", params=params) as resp:
                if (
                        resp and resp.status == 200 and
                        resp.content.total_bytes != 2  # b'[]'
                ):
                    bmaps = await resp.json()

                    embed = Embed(
                        title=f"Pick a map to edit status on.",
                        timestamp=datetime.now(),
                        color=0xeaff00
                    )

                    description = ""
                    for idx, bmap in enumerate(bmaps['set']):
                        description += f"`#{idx + 1}.` **[{bmap['version']}]** - {convert_status_str(int(bmap['status']))}\n"

                    embed.description = description
                    emb_mess = await ctx.send("**Send position in chat to pick a map.**", embed=embed)

            valid = False
            while valid is False:
                try:
                    reply = await self.bot.wait_for('message', check=lambda x: x.channel == ctx.channel and x.author == ctx.author and x.content.isdecimal(),
                                                               timeout=60.0)
                except asyncio.TimeoutError:
                    msg = await ctx.send('Time is up!')

                    await msg.delete(delay=15)
                    await emb_mess.delete(delay=15)
                    return
                else:
                    reply.content = int(reply.content)
                    if reply.content > len(bmaps) or reply.content <= 0:
                        msg = await ctx.send('Specified position is out of range.')

                        await reply.delete(delay=15)
                        await msg.delete(delay=15)
                    else:
                        if (bm_status := bmaps['set'][reply.content - 1]['status']) == convert_str_status(status):
                            msg = await ctx.send(f"This map is already {convert_status_str(bm_status)}")

                            await msg.delete(delay=15)
                            await reply.delete(delay=15)
                        else:
                            await reply.delete()
                            await emb_mess.delete()

                            valid = True

            params = {
                "secret": config.API_SECRET,
                "action": "status_map",
                "admin": admin['safe_name'],
                "map_id": bmaps['set'][reply.content - 1]['id'],
                "status": status
            }
            async with glob.http.get("https://osu.sakuru.pw/api/handle_admin", params=params) as resp:
                if resp.status == 200:
                    await ctx.message.add_reaction('\N{OK HAND SIGN}')
                else:
                    pass

        elif type =="set":
            params = {
                "set_id": req['beatmap']['set_id']
            }

            async with glob.http.get("https://osu.sakuru.pw/api/get_map_info", params=params) as resp:
                if (
                        resp and resp.status == 200 and
                        resp.content.total_bytes != 2  # b'[]'
                ):
                    bmaps = await resp.json()

                    if all([x['status'] == convert_str_status(status) for x in bmaps['set']]):
                        msg = await ctx.send(f"This set is already {convert_status_str(bmaps['set'][0]['status'])}")

                        await ctx.message.delete(delay=15)
                        await msg.delete(delay=15)
                        return

            params = {
                "secret": config.API_SECRET,
                "action": "status_set",
                "admin": admin['safe_name'],
                "set_id": req['beatmap']['set_id'],
                "status": status
            }

            async with glob.http.get("https://osu.sakuru.pw/api/handle_admin", params=params) as resp:
                if resp.status == 200:
                    await ctx.message.add_reaction('\N{OK HAND SIGN}')
                else:
                    pass

    @sakuroCommand(hidden=True)
    @commands.check(sakuru_only)
    @commands.has_permissions(ban_members=True)
    async def rqclose(self, ctx: ContextWrap):
        if (
            not isinstance(ctx.message.channel, Thread) or
            not ctx.message.channel.parent_id == config.MAP_REQS
        ):
            return

        if ctx.message.channel.archived:
            return

        req_table = glob.db.table('map_reqs')

        Requests = Query()
        req = req_table.get((Requests.thread_id == ctx.message.channel.id) & (Requests.active == True))

        req_table.update(
            dbset('active', False),
            doc_ids=[req.doc_id]
        )

        first_message = await ctx.message.channel.parent.fetch_message(req['original_id'])

        await first_message.delete()
        await ctx.channel.delete()

    @sakuroCommand(hidden=True)
    @commands.check(sakuru_only)
    @commands.has_permissions(ban_members=True)
    async def rqreject(self, ctx: ContextWrap, *reason: str):
        if (
            not isinstance(ctx.message.channel, Thread) or
            not ctx.message.channel.parent_id == config.MAP_REQS
        ):
            return

        if ctx.message.channel.archived:
            return

        req_table = glob.db.table('map_reqs')

        Requests = Query()
        req = req_table.get((Requests.thread_id == ctx.message.channel.id) & (Requests.active == True))

        first_message = await ctx.message.channel.parent.fetch_message(req['original_id'])
        requester = ctx.guild.get_member(req['requester'])

        params = {
            "id": req['beatmap']['id']
        }
        async with glob.http.get("https://osu.sakuru.pw/api/get_map_info", params=params) as resp:
            if (
                resp and resp.status == 200 and
                resp.content.total_bytes != 2  # b'[]'
            ):
                bmap = (await resp.json())['map']
                embed = Embed(
                    title=f"Map Request: {bmap['artist']} - {bmap['title']}",
                    color=ctx.author.color,
                    description=f"Your request has been rejected!\n**Reason:** `{' '.join(reason)}`\n\n**Nominator:** {ctx.author.mention}",
                    timestamp=datetime.now()
                )

                embed.set_footer(text="Sakuru.pw osu! Private Server.")
                embed.set_thumbnail(url=ctx.author.avatar.url)

                await requester.send(embed=embed)

        req_table.update(
            dbset('active', False),
            doc_ids=[req.doc_id]
        )

        await first_message.delete()
        await ctx.channel.delete()

def setup(bot):
    log(f"Initiated {__name__} cog!", Ansi.CYAN)
    bot.add_cog(AdminCog(bot))


#
# Archives, to the specified folder, the logged output generated by a benchmark
# run.
#
# @author A. Shawn Bandy
import os
import zipfile
import datetime
import requests
# Follows closely from:
# http://stackoverflow.com/a/34153816
#
# Paths to the log folders are generated by TFB and where those files
# should be archived.
#
path_in = os.path.abspath(os.path.normpath(os.path.expanduser(os.path.join( \
    os.environ['TFB_REPOPARENT'], os.environ['TFB_REPONAME'], \
    'results'))))
date_time = datetime.datetime.now()
dt_folder = date_time.strftime('%Y%m%d%H%M%S')
path_out = os.path.abspath(os.path.join(os.environ['TFB_LOGSFOLDER'], \
    dt_folder))

if not os.path.exists(path_out):
    os.makedirs(path_out)

zip_path = path_out + '/results.zip'

zip_file = zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED)

for root, dirs, files in os.walk(path_in):
    for file in files:
        zip_file.write(os.path.join(root, file))

zip_file.close()

results_upload_uri = os.environ['TFB_UPLOADURI']

if results_upload_uri != None:
    with open(zip_path, 'rb') as file_to_upload:
        requests.post(
            results_upload_uri,
            headers={'Content-Type': 'application/zip'},
            data=file_to_upload)


from django.contrib import admin
from .models import Project, Course, Message, Demos
from django.contrib.auth.models import Group

class CourseAdmin(admin.ModelAdmin):
	list_display = ('title', 'category', 'date_created', 'link')

class ProjectAdmin(admin.ModelAdmin):
	list_display = ('title', 'date_created', 'link')

class MessageAdmin(admin.ModelAdmin):
	list_display = ('email', 'name', 'text', 'received', 'replied', 'date_created')

class DemosAdmin(admin.ModelAdmin):
	list_display = ('title', 'demo_url')

admin.site.register(Course, CourseAdmin)
admin.site.register(Project, ProjectAdmin)
admin.site.register(Message, MessageAdmin)
admin.site.register(Demos, DemosAdmin)
admin.site.unregister(Group)


from gpflow.kernels import Kernel
from gpflow.utilities import positive
from gpflow import Parameter
import tensorflow as tf
from tensorflow_probability import bijectors as tfb

class Batch_simple_SSK(Kernel):
    """
   with hyperparameters:
    1) match_decay float
        decrease the contribution of long subsequences
    3) max_subsequence_length int
        largest subsequence considered
    """

    def __init__(self,active_dims=[0],decay=0.1,max_subsequence_length=3,
                 alphabet = [], maxlen=0, batch_size=100):
        super().__init__(active_dims=active_dims)
        # constrain decay kernel params to between 0 and 1
        self.logistic = tfb.Chain([tfb.Shift(tf.cast(0,tf.float64))(tfb.Scale(tf.cast(1,tf.float64))),tfb.Sigmoid()])
        self.decay_param= Parameter(decay, transform=self.logistic ,name="decay")

        # use will use copies of the kernel params to stop building expensive computation graph
        # we instead efficientely calculate gradients using dynamic programming
        # These params are updated at every call to K and K_diag (to check if parameters have been updated)
        self.decay = self.decay_param.numpy()

        self.decay_unconstrained = self.decay_param.unconstrained_variable.numpy()

        self.order_coefs=tf.ones(max_subsequence_length,dtype=tf.float64)

        # store additional kernel parameters
        self.max_subsequence_length = tf.constant(max_subsequence_length)
        self.alphabet =  tf.constant(alphabet)
        self.alphabet_size=tf.shape(self.alphabet)[0]
        self.maxlen =  tf.constant(maxlen)
        self.batch_size = tf.constant(batch_size)

        # build a lookup table of the alphabet to encode input strings
        self.table = tf.lookup.StaticHashTable(
            initializer=tf.lookup.KeyValueTensorInitializer(
                keys=tf.constant(["PAD"]+alphabet),
                values=tf.constant(range(0,len(alphabet)+1)),),default_value=0)

        # initialize helful construction matricies to be lazily computed once needed
        self.D = None
        self.dD_dgap = None

    def K_diag(self, X):
        r"""
        The diagonal elements of the string kernel are always unity (due to normalisation)
        """
        return tf.ones(tf.shape(X)[:-1],dtype=tf.float64)

    def K(self, X1, X2=None):
        r"""
        Vectorized kernel calc.
        Following notation from Beck (2017), i.e have tensors S,D,Kpp,Kp
        Input is two tensors of shape (# strings , # characters)
        and we calc the pair-wise kernel calcs between the elements (i.e n kern calcs for two lists of length n)
        D is the tensor than unrolls the recursion and allows vecotrizaiton
        """

        # Turn our inputs into lists of integers using one-hot embedding
        # first split up strings and pad to fixed length and prep for gpu
        # pad until all have length of self.maxlen
        # turn into one-hot  i.e. shape (# strings, #characters+1, alphabet size)
        X1 = tf.strings.split(tf.squeeze(X1,1)).to_tensor("PAD",shape=[None,self.maxlen])
        X1 = self.table.lookup(X1)
        # keep track of original input sizes
        X1_shape = tf.shape(X1)[0]
        X1 = tf.one_hot(X1,self.alphabet_size+1,dtype=tf.float64)
        if X2 is None:
            X2 = X1
            X2_shape = X1_shape
            self.symmetric = True
        else:
            self.symmetric = False
            X2 = tf.strings.split(tf.squeeze(X2,1)).to_tensor("PAD",shape=[None,self.maxlen])
            X2 = self.table.lookup(X2)
            X2_shape = tf.shape(X2)[0]
            X2 = tf.one_hot(X2,self.alphabet_size+1,dtype=tf.float64)

        # prep the decay tensors
        self._precalc()

        # combine all target strings and remove the ones in the first column that encode the padding (i.e we dont want them to count as a match)
        X_full = tf.concat([X1,X2],0)[:,:,1:]

        # get indicies of all possible pairings from X and X2
        # this way allows maximum number of kernel calcs to be squished onto the GPU (rather than just doing individual rows of gram)
        indicies_2, indicies_1 = tf.meshgrid(tf.range(0, X1_shape ),tf.range(X1_shape , tf.shape(X_full)[0]))
        indicies = tf.concat([tf.reshape(indicies_1,(-1,1)),tf.reshape(indicies_2,(-1,1))],axis=1)
        if self.symmetric:
            # if symmetric then only calc upper matrix (fill in rest later)
            indicies = tf.boolean_mask(indicies,tf.greater_equal(indicies[:,1]+ X1_shape ,indicies[:,0]))
        else:
            # if not symmetric need to calculate some extra kernel evals for the normalization later on
            indicies = tf.concat([indicies,tf.tile(tf.expand_dims(tf.range(tf.shape(X_full)[0]),1),(1,2))],0)

        # make kernel calcs in batches
        num_batches = tf.cast(tf.math.ceil(tf.shape(indicies)[0]/self.batch_size),dtype=tf.int32)
        k_split =  tf.TensorArray(tf.float64, size=num_batches,clear_after_read=False,infer_shape=False)

        # iterate through batches
        for j in tf.range(num_batches):
            # collect strings for this batch
            indicies_batch = indicies[self.batch_size*j:self.batch_size*(j+1)]
            X_batch = tf.gather(X_full,indicies_batch[:,0],axis=0)
            X2_batch = tf.gather(X_full,indicies_batch[:,1],axis=0)

            # Make S: the similarity tensor of shape (# strings, #characters, # characters)
            #S = tf.matmul( tf.matmul(X_batch,self.sim),tf.transpose(X2_batch,perm=(0,2,1)))
            S = tf.matmul(X_batch,tf.transpose(X2_batch,perm=(0,2,1)))
            # collect results for the batch
            result = self.kernel_calc(S)
            k_split = k_split.write(j,result)

        # combine batch results
        k = tf.expand_dims(k_split.concat(),1)
        k_split.close()

        # put results into the right places in the gram matrix and normalize
        if self.symmetric:
            # if symmetric then only put in top triangle (inc diag)
            mask = tf.linalg.band_part(tf.ones((X1_shape,X2_shape),dtype=tf.int64), 0, -1)
            non_zero = tf.not_equal(mask, tf.constant(0, dtype=tf.int64))

            # Extracting the indices of upper triangle elements
            indices = tf.where(non_zero)
            out = tf.SparseTensor(indices,tf.squeeze(k),dense_shape=tf.cast((X1_shape,X2_shape),dtype=tf.int64))
            k_results = tf.sparse.to_dense(out)

            # add in mising elements (lower diagonal)
            k_results = k_results + tf.linalg.set_diag(tf.transpose(k_results),tf.zeros(X1_shape,dtype=tf.float64))

            # normalise
            X_diag_Ks = tf.linalg.diag_part(k_results)
            norm = tf.tensordot(X_diag_Ks, X_diag_Ks,axes=0)
            k_results = tf.divide(k_results, tf.sqrt(norm))
        else:

            # otherwise can just reshape into gram matrix
            # but first take extra kernel calcs off end of k and use them to normalise
            X_diag_Ks = tf.reshape(k[X1_shape*X2_shape:X1_shape*X2_shape+X1_shape],(-1,))
            X2_diag_Ks = tf.reshape(k[-X2_shape:],(-1,))
            k = k[0:X1_shape*X2_shape]
            k_results = tf.transpose(tf.reshape(k,[X2_shape,X1_shape]))
            # normalise
            norm = tf.tensordot(X_diag_Ks, X2_diag_Ks,axes=0)
            k_results = tf.divide(k_results, tf.sqrt(norm))

        return k_results

    def _precalc(self):
        r"""
        Update stored kernel params (incase they have changed)
        and precalc D and dD_dgap as required for kernel calcs
        following notation from Beck (2017)
        """
        self.decay = self.decay_param.numpy()
        self.decay_unconstrained = self.decay_param.unconstrained_variable.numpy()

        tril =  tf.linalg.band_part(tf.ones((self.maxlen,self.maxlen),dtype=tf.float64), -1, 0)
        # get upper triangle matrix of increasing intergers
        values = tf.TensorArray(tf.int32, size= self.maxlen)
        for i in tf.range(self.maxlen):
            values = values.write(i,tf.range(-i-1,self.maxlen-1-i))
        power = tf.cast(values.stack(),tf.float64)
        values.close()
        power = tf.linalg.band_part(power, 0, -1) - tf.linalg.band_part(power, 0, 0) + tril
        tril = tf.transpose(tf.linalg.band_part(tf.ones((self.maxlen,self.maxlen),dtype=tf.float64), -1, 0))-tf.eye(self.maxlen,dtype=tf.float64)
        gaps = tf.fill([self.maxlen, self.maxlen],self.decay)

        self.D = tf.pow(gaps*tril, power)
        self.dD_dgap = tf.pow((tril * gaps), (power - 1.0)) * tril * power

    @tf.custom_gradient
    def kernel_calc(self,S):

        # fake computations to ensure we take the custom gradients for these two params
        a = tf.square(self.decay_param)

        if self.symmetric:
            k, dk_dgap = tf.stop_gradient(self.kernel_calc_with_grads(S))
        else:
            k = tf.stop_gradient(self.kernel_calc_without_grads(S))

        def grad(dy, variables=None):
            # get gradients of unconstrained params
            grads= {}
            if self.symmetric:
                grads['decay:0'] = tf.reduce_sum(tf.multiply(dy,dk_dgap*tf.math.exp(self.logistic.forward_log_det_jacobian(self.decay_unconstrained,0))))
                gradient = [grads[v.name] for v in variables]
            else:
                gradient = [None for v in variables]
            return ((None),gradient)

        return k, grad

    def kernel_calc_without_grads(self,S):

        # store squared match coef for easier calc later
        match_sq = tf.square(self.decay)

        # calc subkernels for each subsequence length (See Moss et al. 2020 for notation)
        Kp = tf.TensorArray(tf.float64,size=self.max_subsequence_length,clear_after_read=False)

        # fill in first entries
        Kp = Kp.write(0, tf.ones(shape=tf.stack([tf.shape(S)[0], self.maxlen,self.maxlen]), dtype=tf.float64))

        # calculate dynamic programs
        for i in tf.range(self.max_subsequence_length-1):
            Kp_temp = tf.multiply(S, Kp.read(i))
            Kp_temp0 =  match_sq * Kp_temp
            Kp_temp1 = tf.matmul(Kp_temp0,self.D)
            Kp_temp2 = tf.matmul(self.D,Kp_temp1,transpose_a=True)
            Kp = Kp.write(i+1,Kp_temp2)

        # Final calculation. We gather all Kps
        Kp_stacked = Kp.stack()
        Kp.close()

        # combine and get overall kernel
        aux = tf.multiply(S, Kp_stacked)
        aux = tf.reduce_sum(aux, -1)
        sum2 = tf.reduce_sum(aux, -1)
        Ki = sum2 * match_sq
        k = tf.linalg.matvec(tf.transpose(Ki),self.order_coefs)

        return k

    def kernel_calc_with_grads(self,S):
        # store squared match coef for easier calc later
        match_sq = tf.square(self.decay)
        # calc subkernels for each subsequence length (See Moss et al. 2020 for notation)
        Kp = tf.TensorArray(tf.float64,size=self.max_subsequence_length,clear_after_read=False)
        dKp_dgap = tf.TensorArray(tf.float64, size=self.max_subsequence_length, clear_after_read=False)

        # fill in first entries
        Kp = Kp.write(0, tf.ones(shape=tf.stack([tf.shape(S)[0], self.maxlen,self.maxlen]), dtype=tf.float64))
        dKp_dgap = dKp_dgap.write(0, tf.zeros(shape=tf.stack([tf.shape(S)[0], self.maxlen,self.maxlen]), dtype=tf.float64))

        # calculate dynamic programs
        for i in tf.range(self.max_subsequence_length-1):
            Kp_temp = tf.multiply(S, Kp.read(i))
            Kp_temp0 =  match_sq * Kp_temp
            Kp_temp1 = tf.matmul(Kp_temp0,self.D)
            Kp_temp2 = tf.matmul(self.D,Kp_temp1,transpose_a=True)
            Kp = Kp.write(i+1,Kp_temp2)

            dKp_dgap_temp_1 =  tf.matmul(self.dD_dgap,Kp_temp1,transpose_a=True)
            dKp_dgap_temp_2 =  tf.multiply(S, dKp_dgap.read(i))
            dKp_dgap_temp_2 = dKp_dgap_temp_2 * match_sq
            dKp_dgap_temp_2 = tf.matmul(dKp_dgap_temp_2,self.D)
            dKp_dgap_temp_2 = dKp_dgap_temp_2 + tf.matmul(Kp_temp0,self.dD_dgap)
            dKp_dgap_temp_2 = tf.matmul(self.D,dKp_dgap_temp_2,transpose_a=True)
            dKp_dgap = dKp_dgap.write(i+1,dKp_dgap_temp_1 + dKp_dgap_temp_2)

        # Final calculation. We gather all Kps
        Kp_stacked = Kp.stack()
        Kp.close()
        dKp_dgap_stacked = dKp_dgap.stack()
        dKp_dgap.close()

        # combine and get overall kernel

        # get k
        aux = tf.multiply(S, Kp_stacked)
        aux = tf.reduce_sum(aux, -1)
        sum2 = tf.reduce_sum(aux, -1)
        Ki = sum2 * match_sq
        k = tf.linalg.matvec(tf.transpose(Ki),self.order_coefs)

        # get gap decay grads
        temp = tf.multiply(S, dKp_dgap_stacked)
        temp = tf.reduce_sum(temp, -1)
        temp = tf.reduce_sum(temp, -1)
        temp = temp * match_sq
        dk_dgap = tf.linalg.matvec(tf.transpose(temp),self.order_coefs)

        return k, dk_dgap


"""
Compat. layer between LWR and Galaxy.
"""


import unittest
import swift_idl as IDL

test_structure = {
    "key.kind" : "source.lang.swift.decl.struct",
    "key.offset" : 19,
    "key.nameoffset" : 26,
    "key.namelength" : 3,
    "key.inheritedtypes" : [
        {
            "key.name" : "JSONDecodable"
        }
    ],
    "key.bodylength" : 110,
    "key.accessibility" : "source.lang.swift.accessibility.internal",
    "key.substructure" : [
        {
            "key.kind" : "source.lang.swift.decl.var.instance",
            "key.offset" : 72,
            "key.attributes" : [
                {
                    "key.attribute" : "source.decl.attribute.__raw_doc_comment"
                }
            ],
            "key.nameoffset" : 76,
            "key.namelength" : 2,
            "key.length" : 15,
            "key.accessibility" : "source.lang.swift.accessibility.internal",
            "key.substructure" : [

            ],
            "key.typename" : "Int",
            "key.name" : "id"
        },
        {
            "key.kind" : "source.lang.swift.decl.var.instance",
            "key.offset" : 92,
            "key.nameoffset" : 96,
            "key.namelength" : 5,
            "key.length" : 17,
            "key.accessibility" : "source.lang.swift.accessibility.internal",
            "key.substructure" : [

            ],
            "key.typename" : "String",
            "key.name" : "query"
        },
        {
            "key.kind" : "source.lang.swift.decl.var.instance",
            "key.offset" : 126,
            "key.attributes" : [
                {
                    "key.attribute" : "source.decl.attribute.__raw_doc_comment"
                }
            ],
            "key.nameoffset" : 130,
            "key.namelength" : 1,
            "key.length" : 13,
            "key.accessibility" : "source.lang.swift.accessibility.internal",
            "key.substructure" : [

            ],
            "key.typename" : "String",
            "key.name" : "z"
        }
    ],
    "key.name" : "Foo",
    "key.length" : 138,
    "key.bodyoffset" : 46
}

test_syntax = [
  { "offset" : 0, "length" : 6, "type" : "source.lang.swift.syntaxtype.keyword" },
  { "offset" : 7, "length" : 10, "type" : "source.lang.swift.syntaxtype.identifier" },
  { "offset" : 19, "length" : 6, "type" : "source.lang.swift.syntaxtype.keyword" },
  { "offset" : 26, "length" : 3, "type" : "source.lang.swift.syntaxtype.identifier" },
  { "offset" : 31, "length" : 13, "type" : "source.lang.swift.syntaxtype.typeidentifier" },
  { "offset" : 47, "length" : 21, "type" : "source.lang.swift.syntaxtype.comment" },
  { "offset" : 72, "length" : 3, "type" : "source.lang.swift.syntaxtype.keyword" },
  { "offset" : 76, "length" : 2, "type" : "source.lang.swift.syntaxtype.identifier" },
  { "offset" : 80, "length" : 3, "type" : "source.lang.swift.syntaxtype.typeidentifier" },
  { "offset" : 86, "length" : 1, "type" : "source.lang.swift.syntaxtype.number" },
  { "offset" : 92, "length" : 3, "type" : "source.lang.swift.syntaxtype.keyword" },
  { "offset" : 96, "length" : 5, "type" : "source.lang.swift.syntaxtype.identifier" },
  { "offset" : 103, "length" : 6, "type" : "source.lang.swift.syntaxtype.typeidentifier" },
  { "offset" : 110, "length" : 12, "type" : "source.lang.swift.syntaxtype.comment" },
  { "offset" : 126, "length" : 3, "type" : "source.lang.swift.syntaxtype.keyword" },
  { "offset" : 130, "length" : 1, "type" : "source.lang.swift.syntaxtype.identifier" },
  { "offset" : 133, "length" : 6, "type" : "source.lang.swift.syntaxtype.typeidentifier" },
  { "offset" : 144, "length" : 12, "type" : "source.lang.swift.syntaxtype.comment" }
]

test_source = '''import Foundation

struct Foo: JSONDecodable { // sample:"foo,,bar"
    let id: Int = 3
    let query: String // json:"q"
    let z: String     // json:"-"
}
'''

class SampleStructTest(unittest.TestCase):
    def test_getSwiftTokens(self):
        tk = IDL.getSwiftTokens(test_syntax, test_source)
        self.assertEqual('import', tk[0].content)
        self.assertEqual(1, tk[0].line)
        self.assertEqual('source.lang.swift.syntaxtype.keyword', tk[0].tokenType)

        self.assertEqual('}\n', tk[-1].content)
        self.assertEqual(7, tk[-1].line)
        self.assertEqual('omittedtoken', tk[-1].tokenType)

if __name__ == '__main__':
    unittest.main()


from os import getcwd
import sys

sys.path.append(getcwd() + '/..')  # Add src/ dir to import path
import traceback
import logging
from os.path import join
from itertools import combinations

import networkx as nx
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

import libs.osLib as ol

def removeDiagonal(A):
    m = A.shape[0]
    strided = np.lib.stride_tricks.as_strided
    s0,s1 = A.strides
    return strided(A.ravel()[1:], shape=(m-1, m), strides=(s0+s1, s1)).reshape(m, -1)

if __name__ == '__main__':

    root = logging.getLogger()
    root.setLevel(logging.DEBUG)

    baseDir, outputDir = '../../data/adjacencyMatrices', '../../data/plots'
    loadNodeMappings, loadAdjacencies = True, False

    numClusters = 2

    classMapping = {
        'time': 'T',
        'content': 'C',
        'tag': 'G',
        'location': 'L',
    }

    try:
        # metapaths = [['time', 'content', 'time'], ['tag', 'content', 'tag'], ['location', 'content', 'location'] ] # ['time', 'content', 'time']    #     #
        metapaths = [['time', 'content', 'time']]
        metapaths = [[classMapping[t] for t in metapath] for metapath in metapaths]

        for metapath in metapaths:
            nodeMapping = ol.loadPickle(join(baseDir, f'nodeMapping.pickle'))

            # PathSim load
            similarityM = ol.loadSparce(join(baseDir, f'similarity-{"".join(metapath)}.npz')).toarray()

            # Sclump load
            # similarityM = ol.loadNumpy(join(baseDir, f'SClump-similarity.npy'))

            similarityM = removeDiagonal(similarityM)   # Carefull here - we're removing the relation with itself but breaking the mapping from nodeMapping

            # Remove all zeros
            print(f'Orig shape: {similarityM.shape}')
            similarityM = similarityM[~np.all(similarityM == 0, axis=1)]
            similarityM = similarityM[:, ~np.all(similarityM == 0, axis=0)]
            print(f'Without zeros shape: {similarityM.shape}')

            # Plot simple value histogram
            flattenSim = pd.Series(similarityM.flatten())
            g = sns.distplot(flattenSim, kde=False, bins=10)
            g.set_yscale('log')
            plt.savefig(join(outputDir, f'similarityValueDistribution-{"".join(metapath)}.png'))
            plt.title('Value count in Similarity Matrix')
            print(similarityM.max())

            # Count non-zeros per row
            rowCountNonZero = np.count_nonzero(similarityM, axis=1)

            # Count max value per row
            rowCountMax = np.amax(similarityM, 1)

            # Count min value (that's not a zero) per row
            rowCountMinNonZero = np.where(similarityM > 0, similarityM, similarityM.max()).min(1)

            # Count mean value (that's not a zero) per row
            rowCountMeanNonZero = np.true_divide(similarityM.sum(1), (similarityM!=0).sum(1))

            plotDf = None
            for k, x in {
                'Non zeros per row': rowCountNonZero,
                'Max per row': rowCountMax,
                'Mean per row (no zeros)': rowCountMeanNonZero,
                'Min per row (no zeros)': rowCountMinNonZero,
            }.items():
                auxDf = pd.Series(x, name='Count').to_frame()
                auxDf['Measure'] = k
                plotDf = auxDf if plotDf is None else pd.concat([plotDf, auxDf], ignore_index=False)

            # Make boxplot
            fig, ax = plt.subplots(figsize=(15, 15))
            g = sns.boxplot(ax=ax, data=plotDf, x='Measure', y='Count', palette="Set2", showfliers=True, showmeans=True)
            g.set_yscale('log')
            g.set_yticklabels(g.get_yticks(), size=16)
            # g.set_xticklabels(g.get_xticks(), size=16)
            plt.savefig(join(outputDir, f'statsPerRow-log-{"".join(metapath)}.png'))
            plt.close()

            # Make boxplot
            fig, ax = plt.subplots(figsize=(15, 15))
            g = sns.boxplot(ax=ax, data=plotDf, x='Measure', y='Count', palette="Set2", showfliers=False, showmeans=True)
            g.set_yticklabels(g.get_yticks(), size=16)
            # g.set_xticklabels(g.get_xticks(), size=16)
            plt.savefig(join(outputDir, f'statsPerRow-{"".join(metapath)}.png'))
            plt.close()

            # Make violin plots
            fig = plt.figure(figsize=(12, 12))
            gs = fig.add_gridspec(3, 2)

            ax = fig.add_subplot(gs[0, 0])
            sns.violinplot(data=similarityM.flatten())
            ax.set_xlabel("Similarity as is")

            ax = fig.add_subplot(gs[0, 1])
            sns.violinplot(data=rowCountNonZero)
            ax.set_xlabel("Non zeros per row")

            ax = fig.add_subplot(gs[1, 0])
            sns.violinplot(rowCountMeanNonZero)
            ax.set_xlabel("Mean per row (no zeros)")

            ax = fig.add_subplot(gs[1, 1])
            sns.violinplot(rowCountMinNonZero)
            ax.set_xlabel("Min per row (no zeros)")

            ax = fig.add_subplot(gs[2, 0])
            sns.violinplot(data=rowCountMax)
            ax.set_xlabel("Max per row")

            fig.tight_layout()
            plt.savefig(join(outputDir, f'statsViolinPerRow-{"".join(metapath)}.png'))
            plt.close()

            # Plot as matrix
            """
            fig = plt.figure(figsize=(15, 15))
            ax = plt.axes()
            plt.spy(similarityM, precision=0.1, marker='.', markersize=0.05)
            plt.savefig(join(outputDir, f'similarityMatrixPlot-{"".join(metapath)}.png'))
            plt.close()
            """

        # Select top k most similiar or wtv

        # Pick their similarity vectors

        # Plot them

    except Exception as ex:
        print(traceback.format_exc())