##########################################################################
#
#  Copyright (c) 2011-2012, John Haddon. All rights reserved.
#  Copyright (c) 2011-2015, Image Engine Design Inc. All rights reserved.
#
#  Redistribution and use in source and binary forms, with or without
#  modification, are permitted provided that the following conditions are
#  met:
#
#      * Redistributions of source code must retain the above
#        copyright notice, this list of conditions and the following
#        disclaimer.
#
#      * Redistributions in binary form must reproduce the above
#        copyright notice, this list of conditions and the following
#        disclaimer in the documentation and/or other materials provided with
#        the distribution.
#
#      * Neither the name of John Haddon nor the names of
#        any other contributors to this software may be used to endorse or
#        promote products derived from this software without specific prior
#        written permission.
#
#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
#  IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
#  THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
#  PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
#  CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
#  EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
#  PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
#  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
#  LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
#  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
#  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
##########################################################################

# Work around a bug which causes segfaults if uuid is imported after
# PyQt. See here for details :
#
# https://bugs.gentoo.org/show_bug.cgi?id=317557
# http://www.riverbankcomputing.com/pipermail/pyqt/2010-December/028773.html
#
# Using __import__ rather than import so that we don't pollute the GafferUI
# namespace.
__import__( "uuid" )

##########################################################################
# Function to return the C++ address of a wrapped Qt object. This can
# be useful if needing to implement part of the UI in C++ and the rest
# in Python.
##########################################################################

def _qtAddress( o ) :

	import Qt
	if "PyQt" in Qt.__binding__ :
		import sip
		return sip.unwrapinstance( o )
	else :
		return __shiboken().getCppPointer( o )[0]

##########################################################################
# Function to return a wrapped Qt object from the given C++ address.
# This can be useful if needing to implement part of the UI in C++ and
# the rest in Python.
##########################################################################

def _qtObject( address, type ) :

	import Qt
	if "PyQt" in Qt.__binding__ :
		import sip
		return sip.wrapinstance( address, type )
	else :
		return __shiboken().wrapInstance( address, type )

##########################################################################
# Determines if the wrapped Qt object is still valid
# Useful when having to deal with the consequences of C++/Python deletion
# order challeneges, see:
#    https://github.com/GafferHQ/gaffer/pull/3179
##########################################################################

def _qtObjectIsValid( o ) :

	import Qt
	if "PyQt" in Qt.__binding__ :
		import sip
		return not sip.isdeleted( o )
	else :
		return __shiboken().isValid( o )

##########################################################################
# Shiboken lives in a variety of places depending on which PySide it is.
##########################################################################

def __shiboken() :

	import Qt
	assert( "PyQt" not in Qt.__binding__ )

	if Qt.__binding__ == "PySide2" :
		try :
			import PySide2.shiboken2 as shiboken
		except ImportError :
			import shiboken2 as shiboken
	else :
		try :
			import PySide.shiboken
		except ImportError :
			import shiboken

	return shiboken

##########################################################################
# now import our actual functionality
##########################################################################

# Import modules that must be imported before _GafferUI, using __import__
# to avoid polluting the GafferUI namespace.
__import__( "IECore" )
__import__( "Gaffer" )

from ._GafferUI import *

# general ui stuff first

from .Enums import *
from .Widget import Widget
from .LazyMethod import LazyMethod
from .Menu import Menu
from .ContainerWidget import ContainerWidget
from .Window import Window
from .SplitContainer import SplitContainer
from .ListContainer import ListContainer
from .GridContainer import GridContainer
from .MenuBar import MenuBar
from .EventLoop import EventLoop
from .TabbedContainer import TabbedContainer
from .TextWidget import TextWidget
from .NumericWidget import NumericWidget
from .Button import Button
from .MultiLineTextWidget import MultiLineTextWidget
from .Label import Label
from .GLWidget import GLWidget
from .ScrolledContainer import ScrolledContainer
from .PathWidget import PathWidget
from .PathListingWidget import PathListingWidget
from .PathChooserWidget import PathChooserWidget
from .Dialogue import Dialogue
from .PathChooserDialogue import PathChooserDialogue
from .TextInputDialogue import TextInputDialogue
from .Collapsible import Collapsible
from .ColorSwatch import ColorSwatch
from .Slider import Slider
from .ShowURL import showURL
from .Spacer import Spacer
from .BoolWidget import BoolWidget, CheckBox
from .Image import Image
from .ErrorDialogue import ErrorDialogue
from ._Variant import _Variant
from .VectorDataWidget import VectorDataWidget
from .PathVectorDataWidget import PathVectorDataWidget
from .ProgressBar import ProgressBar
from .SelectionMenu import SelectionMenu
from .PathFilterWidget import PathFilterWidget
from .CompoundPathFilterWidget import CompoundPathFilterWidget
from .InfoPathFilterWidget import InfoPathFilterWidget
from .MatchPatternPathFilterWidget import MatchPatternPathFilterWidget
from .FileSequencePathFilterWidget import FileSequencePathFilterWidget
from .BusyWidget import BusyWidget
from .NumericSlider import NumericSlider
from .ColorChooser import ColorChooser
from .ColorChooserDialogue import ColorChooserDialogue
from .MessageWidget import MessageWidget, MessageSummaryWidget
from .NotificationMessageHandler import NotificationMessageHandler
from .MenuButton import MenuButton
from .MultiSelectionMenu import MultiSelectionMenu
from .PopupWindow import PopupWindow
from .ConfirmationDialogue import ConfirmationDialogue
from .DisplayTransform import DisplayTransform
from .Divider import Divider
from . import _Pointer
from .SplineWidget import SplineWidget
from .Bookmarks import Bookmarks
from . import WidgetAlgo

# then all the PathPreviewWidgets. note that the order
# of import controls the order of display.

from .PathPreviewWidget import PathPreviewWidget
from .CompoundPathPreview import CompoundPathPreview
from .DeferredPathPreview import DeferredPathPreview
from .InfoPathPreview import InfoPathPreview
from .HeaderPathPreview import HeaderPathPreview
from .DataPathPreview import DataPathPreview

# then stuff specific to graph uis

from .BackgroundMethod import BackgroundMethod
from .PlugValueWidget import PlugValueWidget
from .StringPlugValueWidget import StringPlugValueWidget
from .NumericPlugValueWidget import NumericPlugValueWidget
from .BoolPlugValueWidget import BoolPlugValueWidget
from .PathPlugValueWidget import PathPlugValueWidget
from .FileSystemPathPlugValueWidget import FileSystemPathPlugValueWidget
from .VectorDataPlugValueWidget import VectorDataPlugValueWidget
from .PathVectorDataPlugValueWidget import PathVectorDataPlugValueWidget
from .FileSystemPathVectorDataPlugValueWidget import FileSystemPathVectorDataPlugValueWidget
from .PlugWidget import PlugWidget
from .PlugLayout import PlugLayout
from .Editor import Editor
from .PythonEditor import PythonEditor
from .GadgetWidget import GadgetWidget
from .GraphEditor import GraphEditor
from .ScriptWindow import ScriptWindow
from .CompoundEditor import CompoundEditor
from .NameWidget import NameWidget
from .NameLabel import NameLabel
from .NodeSetEditor import NodeSetEditor
from .NodeEditor import NodeEditor
from .Layouts import Layouts
from .NodeMenu import NodeMenu
from . import FileMenu
from . import LayoutMenu
from . import EditMenu
from . import UserPlugs
from .Frame import Frame
from .CompoundNumericPlugValueWidget import CompoundNumericPlugValueWidget
from .BoxPlugValueWidget import BoxPlugValueWidget
from .NodeUI import NodeUI
from .StandardNodeUI import StandardNodeUI
from .NodeToolbar import NodeToolbar
from .StandardNodeToolbar import StandardNodeToolbar
from .Viewer import Viewer
from .ColorSwatchPlugValueWidget import ColorSwatchPlugValueWidget
from .ColorPlugValueWidget import ColorPlugValueWidget
from .AboutWindow import AboutWindow
from . import ApplicationMenu
from .BrowserEditor import BrowserEditor
from .Timeline import Timeline
from .MultiLineStringPlugValueWidget import MultiLineStringPlugValueWidget
from .PresetsPlugValueWidget import PresetsPlugValueWidget
from .GraphComponentBrowserMode import GraphComponentBrowserMode
from .ToolPlugValueWidget import ToolPlugValueWidget
from .LabelPlugValueWidget import LabelPlugValueWidget
from .CompoundDataPlugValueWidget import CompoundDataPlugValueWidget
from .LayoutPlugValueWidget import LayoutPlugValueWidget
from . import ScriptNodeUI
from .RefreshPlugValueWidget import RefreshPlugValueWidget
from . import PreferencesUI
from .SplinePlugValueWidget import SplinePlugValueWidget
from .RampPlugValueWidget import RampPlugValueWidget
from .NodeFinderDialogue import NodeFinderDialogue
from .ConnectionPlugValueWidget import ConnectionPlugValueWidget
from .ButtonPlugValueWidget import ButtonPlugValueWidget
from . import ViewUI
from . import ToolUI
from .Playback import Playback
from . import MetadataWidget
from .UIEditor import UIEditor
from . import GraphBookmarksUI
from . import DocumentationAlgo
from . import _PlugAdder
from .Backups import Backups
from .AnimationEditor import AnimationEditor
from . import CompoundNumericNoduleUI
from . import Examples
from .NameValuePlugValueWidget import NameValuePlugValueWidget
from .ShufflePlugValueWidget import ShufflePlugValueWidget
from .ShufflePlugValueWidget import ShufflesPlugValueWidget

# and then specific node uis

from . import DependencyNodeUI
from . import ComputeNodeUI
from . import RandomUI
from . import SpreadsheetUI
from . import ExpressionUI
from . import BoxUI
from . import ReferenceUI
from . import BackdropUI
from . import DotUI
from . import SubGraphUI
from . import SwitchUI
from . import ContextProcessorUI
from . import ContextVariablesUI
from . import DeleteContextVariablesUI
from . import TimeWarpUI
from . import LoopUI
from . import AnimationUI
from . import BoxIOUI
from . import BoxInUI
from . import BoxOutUI
from . import NameSwitchUI
from . import EditScopeUI

# backwards compatibility
## \todo Remove me
Metadata = __import__( "Gaffer" ).Metadata

__import__( "IECore" ).loadConfig( "GAFFER_STARTUP_PATHS", subdirectory = "GafferUI" )


import pdf_to_json as p2j

import json

url = "file:data/multilingual/Arab.URD/Serif_12/udhr_Arab.URD_Serif_12.pdf"
lConverter = p2j.pdf_to_json.pdf_to_json_converter()
lConverter.mImageHashOnly = True
lDict = lConverter.convert(url)
print(json.dumps(lDict, indent=4, ensure_ascii=False, sort_keys=True))


# -*- coding: utf-8 -*-
#
# python-bplsqlparse documentation build configuration file, created by
# sphinx-quickstart on Thu Feb 26 08:19:28 2009.
#
# This file is execfile()d with the current directory set to its containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import datetime
import sys, os

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.append(os.path.abspath('.'))
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '../../'))

import bplsqlparse

# -- General configuration -----------------------------------------------------

# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = ['sphinx.ext.autodoc', 'sphinx.ext.todo', 'sphinx.ext.coverage',
              'sphinx.ext.autosummary']

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = 'python-bplsqlparse'
copyright = '{:%Y}, Andi Albrecht'.format(datetime.date.today())

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = bplsqlparse.__version__
# The full version, including alpha/beta/rc tags.
release = bplsqlparse.__version__

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of documents that shouldn't be included in the build.
#unused_docs = []

# List of directories, relative to source directory, that shouldn't be searched
# for source files.
exclude_trees = []

# The reST default role (used for this markup: `text`) to use for all documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'tango'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# -- Options for HTML output ---------------------------------------------------

# The theme to use for HTML and HTML Help pages.  Major themes that come with
# Sphinx are currently 'default' and 'sphinxdoc'.
#html_theme = 'agogo'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = [os.path.abspath('../')]

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
#html_static_path = ['_static']

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_use_modindex = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = ''

# Output file base name for HTML help builder.
htmlhelp_basename = 'python-sqlparsedoc'

# -- Options for LaTeX output --------------------------------------------------

# The paper size ('letter' or 'a4').
#latex_paper_size = 'letter'

# The font size ('10pt', '11pt' or '12pt').
#latex_font_size = '10pt'

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
  ('index', 'python-bplsqlparse.tex', 'python-bplsqlparse Documentation',
   'Andi Albrecht', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# Additional stuff for the LaTeX preamble.
#latex_preamble = ''

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_use_modindex = True
todo_include_todos = True


#!/usr/bin/env python3
# Copyright (c) 2014-2016 The Bitcoinold Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
'''
assumevalid.py

Test logic for skipping signature validation on blocks which we've assumed
valid (https://github.com/bitcoinold/bitcoinold/pull/9484)

We build a chain that includes and invalid signature for one of the
transactions:

    0:        genesis block
    1:        block 1 with coinbase transaction output.
    2-101:    bury that block with 100 blocks so the coinbase transaction
              output can be spent
    102:      a block containing a transaction spending the coinbase
              transaction output. The transaction has an invalid signature.
    103-2202: bury the bad block with just over two weeks' worth of blocks
              (2100 blocks)

Start three nodes:

    - node0 has no -assumevalid parameter. Try to sync to block 2202. It will
      reject block 102 and only sync as far as block 101
    - node1 has -assumevalid set to the hash of block 102. Try to sync to
      block 2202. node1 will sync all the way to block 2202.
    - node2 has -assumevalid set to the hash of block 102. Try to sync to
      block 200. node2 will reject block 102 since it's assumed valid, but it
      isn't buried by at least two weeks' work.
'''

from test_framework.mininode import *
from test_framework.test_framework import BitcoinoldTestFramework
from test_framework.util import *
from test_framework.blocktools import create_block, create_coinbase
from test_framework.key import CECKey
from test_framework.script import *

class BaseNode(SingleNodeConnCB):
    def __init__(self):
        SingleNodeConnCB.__init__(self)
        self.last_inv = None
        self.last_headers = None
        self.last_block = None
        self.last_getdata = None
        self.block_announced = False
        self.last_getheaders = None
        self.disconnected = False
        self.last_blockhash_announced = None

    def on_close(self, conn):
        self.disconnected = True

    def wait_for_disconnect(self, timeout=60):
        test_function = lambda: self.disconnected
        assert(wait_until(test_function, timeout=timeout))
        return

    def send_header_for_blocks(self, new_blocks):
        headers_message = msg_headers()
        headers_message.headers = [ CBlockHeader(b) for b in new_blocks ]
        self.send_message(headers_message)

class SendHeadersTest(BitcoinoldTestFramework):
    def __init__(self):
        super().__init__()
        self.setup_clean_chain = True
        self.num_nodes = 3

    def setup_network(self):
        # Start node0. We don't start the other nodes yet since
        # we need to pre-mine a block with an invalid transaction
        # signature so we can pass in the block hash as assumevalid.
        self.nodes = []
        self.nodes.append(start_node(0, self.options.tmpdir, ["-debug"]))

    def run_test(self):

        # Connect to node0
        node0 = BaseNode()
        connections = []
        connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], node0))
        node0.add_connection(connections[0])

        NetworkThread().start() # Start up network handling in another thread
        node0.wait_for_verack()

        # Build the blockchain
        self.tip = int(self.nodes[0].getbestblockhash(), 16)
        self.block_time = self.nodes[0].getblock(self.nodes[0].getbestblockhash())['time'] + 1

        self.blocks = []

        # Get a pubkey for the coinbase TXO
        coinbase_key = CECKey()
        coinbase_key.set_secretbytes(b"horsebattery")
        coinbase_pubkey = coinbase_key.get_pubkey()

        # Create the first block with a coinbase output to our key
        height = 1
        block = create_block(self.tip, create_coinbase(height, coinbase_pubkey), self.block_time)
        self.blocks.append(block)
        self.block_time += 1
        block.solve()
        # Save the coinbase for later
        self.block1 = block
        self.tip = block.sha256
        height += 1

        # Bury the block 100 deep so the coinbase output is spendable
        for i in range(100):
            block = create_block(self.tip, create_coinbase(height), self.block_time)
            block.solve()
            self.blocks.append(block)
            self.tip = block.sha256
            self.block_time += 1
            height += 1

        # Create a transaction spending the coinbase output with an invalid (null) signature
        tx = CTransaction()
        tx.vin.append(CTxIn(COutPoint(self.block1.vtx[0].sha256, 0), scriptSig=b""))
        tx.vout.append(CTxOut(49*100000000, CScript([OP_TRUE])))
        tx.calc_sha256()

        block102 = create_block(self.tip, create_coinbase(height), self.block_time)
        self.block_time += 1
        block102.vtx.extend([tx])
        block102.hashMerkleRoot = block102.calc_merkle_root()
        block102.rehash()
        block102.solve()
        self.blocks.append(block102)
        self.tip = block102.sha256
        self.block_time += 1
        height += 1

        # Bury the assumed valid block 2100 deep
        for i in range(2100):
            block = create_block(self.tip, create_coinbase(height), self.block_time)
            block.nVersion = 4
            block.solve()
            self.blocks.append(block)
            self.tip = block.sha256
            self.block_time += 1
            height += 1

        # Start node1 and node2 with assumevalid so they accept a block with a bad signature.
        self.nodes.append(start_node(1, self.options.tmpdir,
                                     ["-debug", "-assumevalid=" + hex(block102.sha256)]))
        node1 = BaseNode()  # connects to node1
        connections.append(NodeConn('127.0.0.1', p2p_port(1), self.nodes[1], node1))
        node1.add_connection(connections[1])
        node1.wait_for_verack()

        self.nodes.append(start_node(2, self.options.tmpdir,
                                     ["-debug", "-assumevalid=" + hex(block102.sha256)]))
        node2 = BaseNode()  # connects to node2
        connections.append(NodeConn('127.0.0.1', p2p_port(2), self.nodes[2], node2))
        node2.add_connection(connections[2])
        node2.wait_for_verack()

        # send header lists to all three nodes
        node0.send_header_for_blocks(self.blocks[0:2000])
        node0.send_header_for_blocks(self.blocks[2000:])
        node1.send_header_for_blocks(self.blocks[0:2000])
        node1.send_header_for_blocks(self.blocks[2000:])
        node2.send_header_for_blocks(self.blocks[0:200])

        # Send 102 blocks to node0. Block 102 will be rejected.
        for i in range(101):
            node0.send_message(msg_block(self.blocks[i]))
        node0.sync_with_ping() # make sure the most recent block is synced
        node0.send_message(msg_block(self.blocks[101]))
        assert_equal(self.nodes[0].getblock(self.nodes[0].getbestblockhash())['height'], 101)

        # Send 3102 blocks to node1. All blocks will be accepted.
        for i in range(2202):
            node1.send_message(msg_block(self.blocks[i]))
        node1.sync_with_ping() # make sure the most recent block is synced
        assert_equal(self.nodes[1].getblock(self.nodes[1].getbestblockhash())['height'], 2202)

        # Send 102 blocks to node2. Block 102 will be rejected.
        for i in range(101):
            node2.send_message(msg_block(self.blocks[i]))
        node2.sync_with_ping() # make sure the most recent block is synced
        node2.send_message(msg_block(self.blocks[101]))
        assert_equal(self.nodes[2].getblock(self.nodes[2].getbestblockhash())['height'], 101)

if __name__ == '__main__':
    SendHeadersTest().main()


from setuptools import find_packages, setup
from pathlib import Path

this_directory = Path(__file__).parent
readme = (this_directory / "README.md").read_text()

setup(
    name='sentencesimilarity',
    packages=find_packages(),
    version='0.1.1',
    description='Calculates semantic similarity between given sentences.',
    long_description= readme,
    long_description_content_type='text/markdown',
    author='osahin',
    author_email = "oguuzhansahiin@gmail.com",
    license='MIT',
    install_requires=['transformers==4.9.2','scikit_learn==0.24.2','torch==1.9.0'],
    setup_requires=['pytest-runner'],
    tests_require=['pytest==4.4.1'],
    test_suite='tests',
)


# -*- coding: utf-8 -*-
import math,string,itertools,fractions,heapq,collections,re,array,bisect

class PublicTransit:

    def distRaw(self, R, C, i1, j1, i2, j2):
        # q = [(i1, j1, 0)]
        # deltas = [(1, 0), (-1, 0), (0, -1), (0, 1)]
        # while q:
        #     i, j, d = q.pop(0)
        #     if i == i2 and j == j2:
        #         return d
        #     for delta in deltas:
        #         ni = i + delta[0]
        #         nj = j + delta[1]
        #         if 0 <= ni < R and 0 <= nj < C:
        #             q.append((ni, nj, d+1))
        # return 1000
        return abs(i1-i2)+abs(j1-j2)

    def distAfterConnect(self, R, C, connect, i1, j1, i2, j2):
        if i1 == i2 and j1 == j2:
            return 0
        return min(self.distRaw(R, C, i1, j1, i2, j2), \
                   self.distRaw(R, C, i1, j1, connect[0], connect[1]) + self.distRaw(R, C, connect[2], connect[3], i2, j2), \
                   self.distRaw(R, C, i1, j1, connect[2], connect[3]) + self.distRaw(R, C, connect[0], connect[1], i2, j2))

    def maxDist(self, R, C, connect):
        res = 1
        for i1 in range(R):
            for j1 in range(C):
                for i2 in range(R-1, -1, -1):
                    for j2 in range(C-1, -1, -1):
                        if abs(i1-i2) + abs(j1-j2) <= res:
                            continue
                        res = max(res, self.distAfterConnect(R, C, connect, i1, j1, i2, j2))
        return res

    def minimumLongestDistance(self, R, C):

        if R <= 0 or C <= 0:
            return 0
        if R*C <= 2:
            return 1

        res = 1000
        for i1 in range(R):
            for j1 in range(C):
                for i2 in range(R):
                    for j2 in range(C):
                        if i1 == i2 and j1 == j2:
                            continue
                        # connect (i, j) and (i2, j2)
                        res = min(res, self.maxDist(R, C, (i1, j1, i2, j2)))

        return res

# CUT begin
# TEST CODE FOR PYTHON {{{
import sys, time, math

def tc_equal(expected, received):
    try:
        _t = type(expected)
        received = _t(received)
        if _t == list or _t == tuple:
            if len(expected) != len(received): return False
            return all(tc_equal(e, r) for (e, r) in zip(expected, received))
        elif _t == float:
            eps = 1e-9
            d = abs(received - expected)
            return not math.isnan(received) and not math.isnan(expected) and d <= eps * max(1.0, abs(expected))
        else:
            return expected == received
    except:
        return False

def pretty_str(x):
    if type(x) == str:
        return '"%s"' % x
    elif type(x) == tuple:
        return '(%s)' % (','.join( (pretty_str(y) for y in x) ) )
    else:
        return str(x)

def do_test(R, C, __expected):
    startTime = time.time()
    instance = PublicTransit()
    exception = None
    try:
        __result = instance.minimumLongestDistance(R, C);
    except:
        import traceback
        exception = traceback.format_exc()
    elapsed = time.time() - startTime   # in sec

    if exception is not None:
        sys.stdout.write("RUNTIME ERROR: \n")
        sys.stdout.write(exception + "\n")
        return 0

    if tc_equal(__expected, __result):
        sys.stdout.write("PASSED! " + ("(%.3f seconds)" % elapsed) + "\n")
        return 1
    else:
        sys.stdout.write("FAILED! " + ("(%.3f seconds)" % elapsed) + "\n")
        sys.stdout.write("           Expected: " + pretty_str(__expected) + "\n")
        sys.stdout.write("           Received: " + pretty_str(__result) + "\n")
        return 0

def run_tests():
    sys.stdout.write("PublicTransit (500 Points)\n\n")

    passed = cases = 0
    case_set = set()
    for arg in sys.argv[1:]:
        case_set.add(int(arg))

    with open("PublicTransit.sample", "r") as f:
        while True:
            label = f.readline()
            if not label.startswith("--"): break

            R = int(f.readline().rstrip())
            C = int(f.readline().rstrip())
            f.readline()
            __answer = int(f.readline().rstrip())

            cases += 1
            if len(case_set) > 0 and (cases - 1) in case_set: continue
            sys.stdout.write("  Testcase #%d ... " % (cases - 1))
            passed += do_test(R, C, __answer)

    sys.stdout.write("\nPassed : %d / %d cases\n" % (passed, cases))

    T = time.time() - 1431783977
    PT, TT = (T / 60.0, 75.0)
    points = 500 * (0.3 + (0.7 * TT * TT) / (10.0 * PT * PT + TT * TT))
    sys.stdout.write("Time   : %d minutes %d secs\n" % (int(T/60), T%60))
    sys.stdout.write("Score  : %.2f points\n" % points)

if __name__ == '__main__':
    run_tests()

# }}}
# CUT end


# coding: utf-8
# Copyright (c) Max-Planck-Institut für Eisenforschung GmbH - Computational Materials Design (CM) Department
# Distributed under the terms of "New BSD License", see the LICENSE file.

import json
import re
import sys

import yaml

environment_file = '.ci_support/environment.yml'
name_mapping_file = '.ci_support/pypi_vs_conda_names.json'

class EnvironmentUpdater:
    def __init__(self, package_name, from_version, to_version):
        """
        Updates the version of a package in the conda environment file.

        Parameters:
            package_name: Name of the package to update as available on PyPI
            from_version: Version the package is before the update
            to_version: Version to which the package should be updated
        """
        self.from_version = from_version
        self.to_version = to_version
        with open(name_mapping_file, 'r') as f:
            self._name_conversion_dict = json.load(f)

        with open(environment_file, 'r') as f:
            self.environment = yaml.safe_load(f)

        self.package_name = self._convert_package_name(package_name)

    def _convert_package_name(self, name):
        if name in self._name_conversion_dict.keys():
            result = self._name_conversion_dict[name]
        else:
            result = name
        return result

    def _update_dependencies(self):
        updated_dependencies = []

        for dep in self.environment['dependencies']:
            updated_dependencies.append(re.sub(
                r'(' + self.package_name + '.*)' + self.from_version,
                r'\g<1>' + self.to_version,
                dep
            ))

        self.environment['dependencies'] = updated_dependencies

    def _write(self):
        with open(environment_file, 'w') as f:
            yaml.safe_dump(self.environment, f)

    def update_dependencies(self):
        """Update the version of the requested dependency in the environment file"""
        self._update_dependencies()
        self._write()

if len(sys.argv) != 7 or not (sys.argv[1] == 'Bump' and sys.argv[3] == 'from' and sys.argv[5] == 'to'):
    raise ValueError(f"Title of a dependabot PR 'Bump <package> from <version> to <version>' expected, "
                     f"but got {' '.join(sys.argv[1:])}")
package_to_update = sys.argv[2]
from_version = sys.argv[4]
to_version = sys.argv[6]

updater = EnvironmentUpdater(package_to_update, from_version, to_version)
updater.update_dependencies()


# coding=utf-8
# Copyright 2022 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
""" Testing suite for the PyTorch ViLT model. """

import unittest

from datasets import load_dataset
from packaging import version

from transformers import ViltConfig, is_torch_available, is_vision_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask

if is_torch_available():
    import torch

    from transformers import (
        MODEL_MAPPING,
        ViltForImageAndTextRetrieval,
        ViltForImagesAndTextClassification,
        ViltForMaskedLM,
        ViltForQuestionAnswering,
        ViltModel,
    )
    from transformers.models.vilt.modeling_vilt import VILT_PRETRAINED_MODEL_ARCHIVE_LIST

if is_vision_available():
    import PIL
    from PIL import Image

    from transformers import ViltProcessor

class ViltModelTester:
    def __init__(
        self,
        parent,
        batch_size=13,
        seq_length=7,
        image_size=30,
        patch_size=2,
        num_channels=3,
        is_training=True,
        use_input_mask=True,
        use_token_type_ids=True,
        use_labels=True,
        vocab_size=99,
        hidden_size=32,
        num_hidden_layers=5,
        num_attention_heads=4,
        intermediate_size=37,
        hidden_act="gelu",
        hidden_dropout_prob=0.1,
        attention_probs_dropout_prob=0.1,
        max_position_embeddings=512,
        type_vocab_size=16,
        type_sequence_label_size=2,
        initializer_range=0.02,
        num_labels=3,
        scope=None,
        modality_type_vocab_size=2,
        add_multiple_images=False,
        num_images=-1,
    ):
        self.parent = parent
        self.batch_size = batch_size
        self.seq_length = seq_length
        self.image_size = image_size
        self.patch_size = patch_size
        self.num_channels = num_channels
        self.is_training = is_training
        self.use_input_mask = use_input_mask
        self.use_token_type_ids = use_token_type_ids
        self.use_labels = use_labels
        self.vocab_size = vocab_size
        self.hidden_size = hidden_size
        self.num_hidden_layers = num_hidden_layers
        self.num_attention_heads = num_attention_heads
        self.intermediate_size = intermediate_size
        self.hidden_act = hidden_act
        self.hidden_dropout_prob = hidden_dropout_prob
        self.attention_probs_dropout_prob = attention_probs_dropout_prob
        self.max_position_embeddings = max_position_embeddings
        self.type_vocab_size = type_vocab_size
        self.type_sequence_label_size = type_sequence_label_size
        self.initializer_range = initializer_range
        self.num_labels = num_labels
        self.scope = scope
        self.modality_type_vocab_size = modality_type_vocab_size
        self.add_multiple_images = add_multiple_images
        self.num_images = num_images
        # we set the expected sequence length (which is used in several tests)
        # this is equal to the seq length of the text tokens + number of image patches + 1 for the CLS token
        self.expected_seq_len = self.seq_length + (self.image_size // self.patch_size) ** 2 + 1

    def prepare_config_and_inputs(self):
        input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size)
        if self.add_multiple_images:
            pixel_values = floats_tensor([self.batch_size, 2, self.num_channels, self.image_size, self.image_size])
        else:
            pixel_values = floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size])

        input_mask = None
        if self.use_input_mask:
            input_mask = random_attention_mask([self.batch_size, self.seq_length])

        token_type_ids = None
        if self.use_token_type_ids:
            token_type_ids = ids_tensor([self.batch_size, self.seq_length], self.type_vocab_size)

        if self.use_labels:
            token_labels = ids_tensor([self.batch_size, self.seq_length], self.num_labels)

        config = self.get_config()

        return (config, input_ids, token_type_ids, input_mask, pixel_values, token_labels)

    def get_config(self):
        return ViltConfig(
            image_size=self.image_size,
            patch_size=self.patch_size,
            num_channels=self.num_channels,
            vocab_size=self.vocab_size,
            hidden_size=self.hidden_size,
            num_hidden_layers=self.num_hidden_layers,
            num_attention_heads=self.num_attention_heads,
            intermediate_size=self.intermediate_size,
            hidden_act=self.hidden_act,
            hidden_dropout_prob=self.hidden_dropout_prob,
            attention_probs_dropout_prob=self.attention_probs_dropout_prob,
            max_position_embeddings=self.max_position_embeddings,
            type_vocab_size=self.type_vocab_size,
            is_decoder=False,
            initializer_range=self.initializer_range,
            num_labels=self.num_labels,
            modality_type_vocab_size=self.modality_type_vocab_size,
            num_images=self.num_images,
        )

    def create_and_check_model(
        self,
        config,
        input_ids,
        token_type_ids,
        input_mask,
        pixel_values,
        token_labels,
    ):
        model = ViltModel(config=config)
        model.to(torch_device)
        model.eval()
        result = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, pixel_values=pixel_values)
        result = model(input_ids, token_type_ids=token_type_ids, pixel_values=pixel_values)
        result = model(input_ids, pixel_values=pixel_values)
        self.parent.assertEqual(
            result.last_hidden_state.shape, (self.batch_size, self.expected_seq_len, self.hidden_size)
        )

    def prepare_config_and_inputs_for_common(self):
        config_and_inputs = self.prepare_config_and_inputs()
        (
            config,
            input_ids,
            token_type_ids,
            input_mask,
            pixel_values,
            token_labels,
        ) = config_and_inputs
        inputs_dict = {
            "input_ids": input_ids,
            "token_type_ids": token_type_ids,
            "attention_mask": input_mask,
            "pixel_values": pixel_values,
        }
        return config, inputs_dict

    def prepare_pixel_values(self):
        return floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size])

@require_torch
class ViltModelTest(ModelTesterMixin, unittest.TestCase):

    all_model_classes = (
        (
            ViltModel,
            ViltForQuestionAnswering,
            ViltForImageAndTextRetrieval,
            ViltForMaskedLM,
        )
        if is_torch_available()
        else ()
    )
    test_pruning = False
    test_headmasking = False
    test_torchscript = False

    # ViltForMaskedLM, ViltForQuestionAnswering and ViltForImagesAndTextClassification require special treatment
    def _prepare_for_class(self, inputs_dict, model_class, return_labels=False):
        inputs_dict = super()._prepare_for_class(inputs_dict, model_class, return_labels=return_labels)

        # if model_class.__name__ == "ViltForNaturalLanguageVisualReasonining":
        #     inputs_dict["pixel_values"] = floats_tensor([self.model_tester.batch_size, self.model_tester.num_images, self.model_tester.num_channels, self.model_tester.image_size, self.model_tester.image_size])

        if return_labels:
            if model_class.__name__ == "ViltForQuestionAnswering":
                inputs_dict["labels"] = torch.zeros(
                    self.model_tester.batch_size, self.model_tester.num_labels, device=torch_device
                )
            elif model_class.__name__ == "ViltForMaskedLM":
                inputs_dict["labels"] = torch.zeros(
                    (self.model_tester.batch_size, self.model_tester.seq_length), dtype=torch.long, device=torch_device
                )
            elif model_class.__name__ == "ViltForImagesAndTextClassification":
                inputs_dict["labels"] = torch.zeros(
                    self.model_tester.batch_size, dtype=torch.long, device=torch_device
                )

        return inputs_dict

    def setUp(self):
        self.model_tester = ViltModelTester(self)
        self.config_tester = ConfigTester(self, config_class=ViltConfig, hidden_size=37)

    def test_config(self):
        self.config_tester.run_common_tests()

    def test_model(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model(*config_and_inputs)

    def test_training(self):
        if not self.model_tester.is_training:
            return

        for model_class in self.all_model_classes:
            config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
            config.return_dict = True

            if model_class.__name__ == "ViltForImagesAndTextClassification":
                config.modality_type_vocab_size = 3

            # ViltForImageAndTextRetrieval doesn't support training for now
            if model_class in [*get_values(MODEL_MAPPING), ViltForImageAndTextRetrieval]:
                continue

            model = model_class(config)
            model.to(torch_device)
            model.train()
            inputs = self._prepare_for_class(inputs_dict, model_class, return_labels=True)
            for k, v in inputs.items():
                print(k, v.shape)
            loss = model(**inputs).loss
            loss.backward()

    def test_training_gradient_checkpointing(self):
        if not self.model_tester.is_training:
            return

        for model_class in self.all_model_classes:
            config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
            config.use_cache = False
            config.return_dict = True

            # ViltForImageAndTextRetrieval doesn't support training for now
            if (
                model_class in [*get_values(MODEL_MAPPING), ViltForImageAndTextRetrieval]
                or not model_class.supports_gradient_checkpointing
            ):
                continue

            model = model_class(config)
            model.to(torch_device)
            model.gradient_checkpointing_enable()
            model.train()
            inputs = self._prepare_for_class(inputs_dict, model_class, return_labels=True)
            loss = model(**inputs).loss
            loss.backward()

    @unittest.skip(
        reason="""VilT samples image tokens from a multinomial distribution, resulting in not deterministic
                            hidden states"""
    )
    def test_save_load(self):
        pass

    @unittest.skip(
        reason="""VilT samples image tokens from a multinomial distribution, resulting in not deterministic
                            hidden states"""
    )
    def test_determinism(self):
        pass

    @unittest.skip(
        reason="""VilT samples image tokens from a multinomial distribution, resulting in not deterministic
                            hidden states"""
    )
    def test_model_outputs_equivalence(self):
        pass

    def test_attention_outputs(self):
        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
        config.return_dict = True

        seq_len = getattr(self.model_tester, "expected_seq_len", None)

        for model_class in self.all_model_classes:
            inputs_dict["output_attentions"] = True
            inputs_dict["output_hidden_states"] = False
            config.return_dict = True
            model = model_class(config)
            model.to(torch_device)
            model.eval()
            with torch.no_grad():
                outputs = model(**self._prepare_for_class(inputs_dict, model_class))
            attentions = outputs.attentions
            if model_class.__name__ == "ViltForImagesAndTextClassification":
                # attentions are a list of length num_images
                # each element contains the attentions of a particular image index
                self.assertEqual(len(attentions), self.model_tester.num_images)
                self.assertEqual(len(attentions[0]), self.model_tester.num_hidden_layers)
            else:
                self.assertEqual(len(attentions), self.model_tester.num_hidden_layers)

            # check that output_attentions also work using config
            del inputs_dict["output_attentions"]
            config.output_attentions = True
            model = model_class(config)
            model.to(torch_device)
            model.eval()
            with torch.no_grad():
                outputs = model(**self._prepare_for_class(inputs_dict, model_class))
            attentions = outputs.attentions
            if model_class.__name__ == "ViltForImagesAndTextClassification":
                # attentions are a list of length num_images
                # each element contains the attentions of a particular image index
                self.assertEqual(len(attentions), self.model_tester.num_images)
                self.assertEqual(len(attentions[0]), self.model_tester.num_hidden_layers)
            else:
                self.assertEqual(len(attentions), self.model_tester.num_hidden_layers)

            if model_class.__name__ == "ViltForImagesAndTextClassification":
                self.assertListEqual(
                    list(attentions[0][0].shape[-3:]),
                    [self.model_tester.num_attention_heads, seq_len, seq_len],
                )
            else:
                self.assertListEqual(
                    list(attentions[0].shape[-3:]),
                    [self.model_tester.num_attention_heads, seq_len, seq_len],
                )
            out_len = len(outputs)

            # Check attention is always last and order is fine
            inputs_dict["output_attentions"] = True
            inputs_dict["output_hidden_states"] = True
            model = model_class(config)
            model.to(torch_device)
            model.eval()
            with torch.no_grad():
                outputs = model(**self._prepare_for_class(inputs_dict, model_class))

            self.assertEqual(out_len + 1, len(outputs))

            self_attentions = outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions

            if model_class.__name__ == "ViltForImagesAndTextClassification":
                self.assertEqual(len(self_attentions), self.model_tester.num_images)
                self.assertEqual(len(self_attentions[0]), self.model_tester.num_hidden_layers)
                self.assertListEqual(
                    list(self_attentions[0][0].shape[-3:]),
                    [self.model_tester.num_attention_heads, seq_len, seq_len],
                )
            else:
                self.assertEqual(len(self_attentions), self.model_tester.num_hidden_layers)
                self.assertListEqual(
                    list(self_attentions[0].shape[-3:]),
                    [self.model_tester.num_attention_heads, seq_len, seq_len],
                )

    def test_hidden_states_output(self):
        def check_hidden_states_output(inputs_dict, config, model_class):
            model = model_class(config)
            model.to(torch_device)
            model.eval()

            with torch.no_grad():
                outputs = model(**self._prepare_for_class(inputs_dict, model_class))

            hidden_states = outputs.encoder_hidden_states if config.is_encoder_decoder else outputs.hidden_states

            expected_num_layers = getattr(
                self.model_tester, "expected_num_hidden_layers", self.model_tester.num_hidden_layers + 1
            )
            if model_class.__name__ == "ViltForImagesAndTextClassification":
                # hidden_states are a list of length num_images
                # each element contains the hidden states of a particular image index
                self.assertEqual(len(hidden_states), self.model_tester.num_images)
                self.assertEqual(len(hidden_states[0]), expected_num_layers)
            else:
                self.assertEqual(len(hidden_states), expected_num_layers)

            seq_length = self.model_tester.expected_seq_len

            if model_class.__name__ == "ViltForImagesAndTextClassification":
                self.assertListEqual(
                    list(hidden_states[0][0].shape[-2:]),
                    [seq_length, self.model_tester.hidden_size],
                )
            else:
                self.assertListEqual(
                    list(hidden_states[0].shape[-2:]),
                    [seq_length, self.model_tester.hidden_size],
                )

        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
            print("Model class:", model_class)
            inputs_dict["output_hidden_states"] = True
            check_hidden_states_output(inputs_dict, config, model_class)

            # check that output_hidden_states also work using config
            del inputs_dict["output_hidden_states"]
            config.output_hidden_states = True

            check_hidden_states_output(inputs_dict, config, model_class)

    def test_retain_grad_hidden_states_attentions(self):
        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
        config.output_hidden_states = True
        config.output_attentions = True

        # no need to test all models as different heads yield the same functionality
        model_class = self.all_model_classes[0]
        model = model_class(config)
        model.to(torch_device)

        inputs = self._prepare_for_class(inputs_dict, model_class)

        outputs = model(**inputs)

        output = outputs[0]

        # Encoder-/Decoder-only models
        hidden_states = outputs.hidden_states[0]
        attentions = outputs.attentions[0]

        if model_class.__name__ == "ViltForImagesAndTextClassification":
            # hidden_states are a list of length num_images
            # each element contains the hidden states of a particular image index
            hidden_states[0].retain_grad()
            attentions[0].retain_grad()
        else:
            hidden_states.retain_grad()
            attentions.retain_grad()

        output.flatten()[0].backward(retain_graph=True)

        if model_class.__name__ == "ViltForImagesAndTextClassification":
            # hidden_states are a list of length num_images
            # each element contains the hidden states of a particular image index
            self.assertIsNotNone(hidden_states[0].grad)
            self.assertIsNotNone(attentions[0].grad)
        else:
            self.assertIsNotNone(hidden_states.grad)
            self.assertIsNotNone(attentions.grad)

    @slow
    def test_model_from_pretrained(self):
        for model_name in VILT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
            model = ViltModel.from_pretrained(model_name)
            self.assertIsNotNone(model)

@require_torch
class ViltForImagesAndTextClassificationModelTest(ViltModelTest, unittest.TestCase):

    all_model_classes = (ViltForImagesAndTextClassification,) if is_torch_available() else ()

    def setUp(self):
        self.model_tester = ViltModelTester(self, modality_type_vocab_size=3, add_multiple_images=True, num_images=2)
        self.config_tester = ConfigTester(self, config_class=ViltConfig, hidden_size=37)

    @unittest.skip("We only test the model that takes in multiple images")
    def test_model(self):
        pass

# We will verify our results on an image of cute cats
def prepare_img():
    image = Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png")
    return image

@require_torch
@require_vision
class ViltModelIntegrationTest(unittest.TestCase):
    @cached_property
    def default_processor(self):
        return ViltProcessor.from_pretrained("dandelin/vilt-b32-finetuned-vqa") if is_vision_available() else None

    @slow
    def test_inference_masked_lm(self):
        model = ViltForMaskedLM.from_pretrained("dandelin/vilt-b32-mlm").to(torch_device)

        processor = self.default_processor
        image = prepare_img()
        text = "a bunch of [MASK] laying on a [MASK]."
        inputs = processor(image, text, return_tensors="pt").to(torch_device)

        # forward pass
        with torch.no_grad():
            outputs = model(**inputs)

        # verify the logits
        expected_shape = torch.Size([1, 11, 30522])
        self.assertEqual(outputs.logits.shape, expected_shape)

        expected_slice = torch.tensor([-12.5061, -12.5123, -12.5174]).to(torch_device)
        self.assertTrue(torch.allclose(outputs.logits[0, 0, :3], expected_slice, atol=1e-4))

        # verify masked token prediction equals "cats"
        predicted_id = outputs.logits[0, 4, :].argmax(-1).item()
        assert processor.decode([predicted_id]) == "cats"

    @slow
    def test_inference_visual_question_answering(self):
        model = ViltForQuestionAnswering.from_pretrained("dandelin/vilt-b32-finetuned-vqa").to(torch_device)

        processor = self.default_processor
        image = prepare_img()
        text = "How many cats are there?"
        inputs = processor(image, text, return_tensors="pt").to(torch_device)

        # forward pass
        with torch.no_grad():
            outputs = model(**inputs)

        # verify the logits
        expected_shape = torch.Size((1, 3129))
        self.assertEqual(outputs.logits.shape, expected_shape)

        expected_slice = torch.tensor([-15.9495, -18.1472, -10.3041]).to(torch_device)

        self.assertTrue(torch.allclose(outputs.logits[0, :3], expected_slice, atol=1e-4))

        # compute loss
        vqa_labels = [[2, 3, 155, 800]]
        vqa_scores = [[1.0, 0.3, 0.3, 0.3]]
        labels = torch.zeros(1, model.config.num_labels).to(torch_device)

        for i, (labels_example, scores_example) in enumerate(zip(vqa_labels, vqa_scores)):
            for l, s in zip(labels_example, scores_example):
                labels[i, l] = s

        # forward pass
        outputs = model(**inputs, labels=labels)

        # verify we have a positive loss
        self.assertTrue(outputs.loss > 0)

    @slow
    def test_inference_natural_language_visual_reasoning(self):
        model = ViltForImagesAndTextClassification.from_pretrained("dandelin/vilt-b32-finetuned-nlvr2").to(
            torch_device
        )

        processor = self.default_processor

        dataset = load_dataset("hf-internal-testing/fixtures_nlvr2", split="test")
        image1 = Image.open(dataset[0]["file"]).convert("RGB")
        image2 = Image.open(dataset[1]["file"]).convert("RGB")

        text = (
            "The left image contains twice the number of dogs as the right image, and at least two dogs in total are"
            " standing."
        )
        encoding_1 = processor(image1, text, return_tensors="pt")
        encoding_2 = processor(image2, text, return_tensors="pt")

        pixel_values = torch.stack([encoding_1.pixel_values, encoding_2.pixel_values], dim=1)

        # forward pass
        outputs = model(
            input_ids=encoding_1.input_ids.to(torch_device),
            pixel_values=pixel_values.to(torch_device),
        )

        # verify the logits
        expected_shape = torch.Size([1, 2])
        self.assertEqual(outputs.logits.shape, expected_shape)

        is_pillow_less_than_9 = version.parse(PIL.__version__) < version.parse("9.0.0")

        if is_pillow_less_than_9:
            expected_slice = torch.tensor(
                [-2.4013, 2.9342],
                device=torch_device,
            )
        else:
            expected_slice = torch.tensor(
                [-2.3713, 2.9168],
                device=torch_device,
            )

        self.assertTrue(torch.allclose(outputs.logits[0, :3], expected_slice, atol=1e-4))


import os
import sys
module_path = os.path.abspath(os.path.join('../models/'))
print(module_path)
if module_path not in sys.path:
    sys.path.append(module_path)

import torch
import torch.nn as nn
import torch.backends.cudnn as cudnn
from torch.autograd import Variable
import numpy as np
import cv2
import time

if torch.cuda.is_available():
    torch.set_default_tensor_type('torch.cuda.FloatTensor')
os.environ["CUDA_VISIBLE_DEVICES"] = "1"
size = 320
# from refinedet import build_refinedet
# from models.multitrident_refinedet_v2 import  build_multitridentrefinedet
from models.multitrident_refinedet import  build_multitridentrefinedet
net = build_multitridentrefinedet('test', size, 21)    # initialize SSD
# net = build_refinedet('test', 512, 21)
# net.load_weights('../weights/RefineDet512_VOC_final.pth')
# net.load_weights('../weights/experiment/320*320/exp_4_[256relufpn][0.3_0.6][mAP_0.77][dilate:11111-12333-12555]/RefineDet320_VOC_275000.pth')
net.load_weights('../weights/experiment/320*320/RefineDet320_VOC_315000.pth')

"""000210 000111 000144 009539 009589 000069 009539 001275 002333 002338 002341
002695 002713 003681 003874 003673 003740"""
im_names = "002695.jpg"

image_file = '/home/yiling/data/VOCdevkit/VOC2007/JPEGImages/' + im_names
image = cv2.imread(image_file, cv2.IMREAD_COLOR)  # uncomment if dataset not download
#%matplotlib inline
from matplotlib import pyplot as plt
from data import VOCDetection, VOC_ROOT, VOCAnnotationTransform
# here we specify year (07 or 12) and dataset ('test', 'val', 'train')
testset = VOCDetection(VOC_ROOT, [('2007', 'val')], None, VOCAnnotationTransform())
img_id = 62
# image = testset.pull_image(img_id)
rgb_image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
# View the sampled input image before transform
plt.figure(figsize=(10,10))
# plt.imshow(rgb_image)
# plt.show()

x = cv2.resize(image, (size, size)).astype(np.float32)
x -= (104.0, 117.0, 123.0)
x = x.astype(np.float32)
x = x[:, :, ::-1].copy()
# plt.imshow(x)
x = torch.from_numpy(x).permute(2, 0, 1)

xx = Variable(x.unsqueeze(0))     # wrap tensor in Variable
if torch.cuda.is_available():
    xx = xx.cuda()
start = time.time()
y = net(xx)
end = time.time()
print(end-start)

from data import VOC_CLASSES as labels
top_k=100

plt.figure(figsize=(10,10))
colors = plt.cm.hsv(np.linspace(0, 1, 21)).tolist()
plt.imshow(rgb_image)  # plot the image for matplotlib
currentAxis = plt.gca()

detections = y.data

# scale each detection back up to the image
scale = torch.Tensor(rgb_image.shape[1::-1]).repeat(2)
for i in range(detections.size(1)):
    for j in range(detections.size(2)):
        if detections[0,i,j,0] > 0.05:
            score = detections[0, i, j, 0]
            label_name = labels[i - 1]
            display_txt = '%s: %.2f' % (label_name, score)
            pt = (detections[0, i, j, 1:] * scale).cpu().numpy()
            coords = (pt[0], pt[1]), pt[2] - pt[0] + 1, pt[3] - pt[1] + 1
            color = colors[i]
            currentAxis.add_patch(plt.Rectangle(*coords, fill=False, edgecolor=color, linewidth=2))
            currentAxis.text(pt[0], pt[1], display_txt, bbox={'facecolor': color, 'alpha': 0.5})
        else:
            continue
    # j = 0
    # while detections[0,i,j,0] >= -1:
    #     score = detections[0,i,j,0]
    #     label_name = labels[i-1]
    #     display_txt = '%s: %.2f'%(label_name, score)
    #     pt = (detections[0,i,j,1:]*scale).cpu().numpy()
    #     coords = (pt[0], pt[1]), pt[2]-pt[0]+1, pt[3]-pt[1]+1
    #     color = colors[i]
    #     currentAxis.add_patch(plt.Rectangle(*coords, fill=False, edgecolor=color, linewidth=2))
    #     currentAxis.text(pt[0], pt[1], display_txt, bbox={'facecolor':color, 'alpha':0.5})
    #     j+=1

plt.show()


# -*- coding: utf-8 -*-
"""
Python Slack Bot class for use with the pythOnBoarding app
"""
import os

from slackclient import SlackClient

# To remember which teams have authorized your app and what tokens are
# associated with each team, we can store this information in memory on
# as a global object. When your bot is out of development, it's best to
# save this in a more persistant memory store.
authed_teams = {}

class Bot(object):
    """ Instanciates a Bot object to handle Slack onboarding interactions."""
    def __init__(self):
        super(Bot, self).__init__()
        self.name = "come_back_here"
        self.emoji = ":robot_face:"
        # When we instantiate a new bot object, we can access the app
        # credentials we set earlier in our local development environment.
        self.oauth = {"client_id": os.environ.get("CLIENT_ID"),
                      "client_secret": os.environ.get("CLIENT_SECRET"),
                      # Scopes provide and limit permissions to what our app
                      # can access. It's important to use the most restricted
                      # scope that your app will need.
                      "scope": "users.profile:read"}
        self.oauth
        self.verification = os.environ.get("VERIFICATION_TOKEN")

        # NOTE: Python-slack requires a client connection to generate
        # an oauth token. We can connect to the client without authenticating
        # by passing an empty string as a token and then reinstantiating the
        # client with a valid OAuth token once we have one.
        self.client = SlackClient("")
        # We'll use this dictionary to store the state of each message object.
        # In a production envrionment you'll likely want to store this more
        # persistantly in  a database.
        self.messages = {}

    def auth(self, code):
        """
        Authenticate with OAuth and assign correct scopes.
        Save a dictionary of authed team information in memory on the bot
        object.

        Parameters
        ----------
        code : str
            temporary authorization code sent by Slack to be exchanged for an
            OAuth token

        """
        # After the user has authorized this app for use in their Slack team,
        # Slack returns a temporary authorization code that we'll exchange for
        # an OAuth token using the oauth.access endpoint
        auth_response = self.client.api_call(
                                "oauth.access",
                                client_id=self.oauth["client_id"],
                                client_secret=self.oauth["client_secret"],
                                code=code
                                )
        # To keep track of authorized teams and their associated OAuth tokens,
        # we will save the team ID and bot tokens to the global
        # authed_teams object
        team_id = auth_response["team_id"]
        authed_teams[team_id] = {"bot_token":
                                 auth_response["access_token"]}
        # Then we'll reconnect to the Slack Client with the correct team's
        # bot token
        self.client = SlackClient(authed_teams[team_id]["bot_token"])

    def bring_back_user(self, user_id, channel, token):
        """
        Create and send an onboarding welcome message to new users. Save the
        time stamp of this message on the message object for updating in the
        future.

        Parameters
        ----------
        team_id : str
            id of the Slack team associated with the incoming event
        user_id : str
            id of the Slack user associated with the incoming event

        """
        # We'll use the message object's method to create the attachments that
        # we'll want to add to our Slack message. This method will also save
        # the attachments on the message object which we're accessing in the
        # API call below through the message object's `attachments` attribute.
        text = "Hey... get back here <@" + str(user_id) + ">"
        self.client.api_call(
                             "chat.postMessage",
                             channel=channel,
                             token=token,
                             username=self.name,
                             icon_emoji=self.emoji,
                             text=text
        )
        self.client.api_call("channels.invite", token=token, channel=channel, user=user_id)


# -*- coding: utf-8 -*-
#########################################################################
#
# Copyright (C) 2016 OSGeo
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
#########################################################################
import errno
import logging

from geoserver.layer import Layer as GsLayer

from django.conf import settings
from django.dispatch import receiver, Signal
from django.forms.models import model_to_dict
from django.contrib.staticfiles.templatetags import staticfiles

# use different name to avoid module clash
from geonode.utils import (
    is_monochromatic_image,
    json_serializer_producer)
from geonode.decorators import on_ogc_backend
from geonode.geoserver.helpers import (
    gs_catalog,
    ogc_server_settings)
from geonode.geoserver.tasks import geoserver_create_thumbnail
from geonode.layers.models import Layer
from geonode.services.enumerations import CASCADED

from . import BACKEND_PACKAGE
from .tasks import geoserver_cascading_delete, geoserver_post_save_layers

logger = logging.getLogger("geonode.geoserver.signals")

geoserver_post_save_complete = Signal(providing_args=['instance'])

def geoserver_delete(typename):
    # cascading_delete should only be called if
    # ogc_server_settings.BACKEND_WRITE_ENABLED == True
    if getattr(ogc_server_settings, "BACKEND_WRITE_ENABLED", True):
        geoserver_cascading_delete.apply_async((typename,))

@on_ogc_backend(BACKEND_PACKAGE)
def geoserver_pre_delete(instance, sender, **kwargs):
    """Removes the layer from GeoServer
    """
    # cascading_delete should only be called if
    # ogc_server_settings.BACKEND_WRITE_ENABLED == True
    if getattr(ogc_server_settings, "BACKEND_WRITE_ENABLED", True):
        if instance.remote_service is None or instance.remote_service.method == CASCADED:
            if instance.alternate:
                geoserver_cascading_delete.apply_async((instance.alternate,))

@on_ogc_backend(BACKEND_PACKAGE)
def geoserver_pre_save(*args, **kwargs):
    # nothing to do here, processing is pushed to post-save
    pass

@on_ogc_backend(BACKEND_PACKAGE)
def geoserver_post_save(instance, sender, created, **kwargs):
    from geonode.messaging import producer
    # this is attached to various models, (ResourceBase, Document)
    # so we should select what will be handled here
    if isinstance(instance, Layer):
        instance_dict = model_to_dict(instance)
        payload = json_serializer_producer(instance_dict)
        try:
            producer.geoserver_upload_layer(payload)
        except Exception as e:
            logger.error(e)
        if getattr(settings, 'DELAYED_SECURITY_SIGNALS', False):
            instance.set_dirty_state()

@on_ogc_backend(BACKEND_PACKAGE)
def geoserver_post_save_local(instance, *args, **kwargs):
    """Send information to geoserver.

       The attributes sent include:

        * Title
        * Abstract
        * Name
        * Keywords
        * Metadata Links,
        * Point of Contact name and url
    """
    geoserver_post_save_layers.apply_async(
        (instance.id, args, kwargs))

@on_ogc_backend(BACKEND_PACKAGE)
def geoserver_pre_save_maplayer(instance, sender, **kwargs):
    # If this object was saved via fixtures,
    # do not do post processing.
    if kwargs.get('raw', False):
        return

    try:
        instance.local = isinstance(
            gs_catalog.get_layer(
                instance.name),
            GsLayer)
    except EnvironmentError as e:
        if e.errno == errno.ECONNREFUSED:
            msg = f'Could not connect to catalog to verify if layer {instance.name} was local'
            logger.warn(msg)
        else:
            raise e

@on_ogc_backend(BACKEND_PACKAGE)
def geoserver_post_save_map(instance, sender, created, **kwargs):
    instance.set_missing_info()
    if not created:
        if not instance.thumbnail_url or \
                instance.thumbnail_url == staticfiles.static(settings.MISSING_THUMBNAIL):
            logger.debug(f"... Creating Thumbnail for Map [{instance.title}]")
            # create_gs_thumbnail(instance, overwrite=False, check_bbox=True)
            geoserver_create_thumbnail.apply_async(((instance.id, False, True, )))

@receiver(geoserver_post_save_complete)
def geoserver_post_save_thumbnail(sender, instance, **kwargs):
    # Creating Layer Thumbnail
    # some thumbnail generators will update thumbnail_url.  If so, don't
    # immediately re-generate the thumbnail here.  use layer#save(update_fields=['thumbnail_url'])
    try:
        instance.refresh_from_db()
        logger.debug(f"... Creating Thumbnail for Layer {instance.title}")
        _recreate_thumbnail = False
        if 'update_fields' in kwargs and kwargs['update_fields'] is not None and \
                'thumbnail_url' in kwargs['update_fields']:
            _recreate_thumbnail = True
        if not instance.thumbnail_url or \
                instance.thumbnail_url == staticfiles.static(settings.MISSING_THUMBNAIL) or \
                is_monochromatic_image(instance.thumbnail_url):
            _recreate_thumbnail = True
        if _recreate_thumbnail:
            geoserver_create_thumbnail.apply_async(((instance.id, False, True, )))
        else:
            logger.debug(f"... Thumbnail for Layer {instance.title} already exists: {instance.thumbnail_url}")
    except Exception as e:
        logger.exception(e)


# -*- coding: utf-8 -*-
#
# dist_fit documentation build configuration file, created by
# sphinx-quickstart on Sat Nov 10 11:16:37 2012.
#
# This file is execfile()d with the current directory set to its containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.
import matplotlib
matplotlib.use('Agg')
import sys
from os.path import dirname, join

sys.path.insert(0, join(dirname(__file__), '../'))
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))

# -- General configuration -----------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = ['matplotlib.sphinxext.only_directives',
                'matplotlib.sphinxext.plot_directive',
                'matplotlib.sphinxext.ipython_directive',
                'sphinx.ext.autodoc',
                'sphinx.ext.mathjax',
                'sphinx.ext.autosummary']

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'probfit'
copyright = u'2012, Piti Ongmongkolkul'
autoclass_content = 'both'
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
import probfit.info
version = probfit.info.__version__
# The full version, including alpha/beta/rc tags.
release = probfit.info.__version__

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build', '_themes']

# The reST default role (used for this markup: `text`) to use for all documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# -- Options for HTML output ---------------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = 'armstrong'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
html_theme_path = ['_themes', ]

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
#html_static_path = ['_static']

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Output file base name for HTML help builder.
htmlhelp_basename = 'probfitdoc'

# -- Options for LaTeX output --------------------------------------------------

latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',

# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',

# Additional stuff for the LaTeX preamble.
#'preamble': '',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
  ('index', 'probfit.tex', u'dist\\_fit Documentation',
   u'Piti Ongmongkolkul', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True

# -- Options for manual page output --------------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    ('index', 'probfit', u'probfit Documentation',
     [u'Piti Ongmongkolkul'], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False

# -- Options for Texinfo output ------------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
  ('index', 'probfit', u'probfit Documentation',
   u'Piti Ongmongkolkul', 'probfit', 'Fitting Stuff',
   'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'


"""
"Beacon" (c) by Ignacio Slater M.
"Beacon" is licensed under a
Creative Commons Attribution 4.0 International License.
You should have received a copy of the license along with this
work. If not, see <http://creativecommons.org/licenses/by/4.0/>.
"""


# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Benchmarks for low-level eager execution primitives.

Packaged as a test to ensure that this code is exercised by continuous
integration tests. To get numbers:

  bazel build -c opt :benchmarks_test &&
  ./bazel-bin/tensorflow/python/eager/benchmarks_test --iters=0
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import argparse
import contextlib
import sys
import time

import numpy as np
from six.moves import xrange  # pylint: disable=redefined-builtin

from tensorflow.python import pywrap_tensorflow
from tensorflow.python.eager import backprop  # pylint: disable=unused-import
from tensorflow.python.eager import context
from tensorflow.python.eager import function
from tensorflow.python.eager import test
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import ops
from tensorflow.python.framework import test_util
from tensorflow.python.ops import gen_math_ops
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import random_ops

FLAGS = None

@contextlib.contextmanager
def timer(label, iters=30000):
  start = time.time()
  yield xrange(iters)
  end = time.time()
  t = (end - start) * 1e6 / iters
  print("%-40s took %.2fus (%d iterations)" % (label, t, iters))

def benchmark_create_tensor(n):
  """Benchmark overheads of creating a Tensor object."""

  def label(s):
    return "{:20s}".format(s)

  with timer(label("np.array([[3.0]])"), iters=n) as iters:
    for _ in iters:
      np.array([[3.0]])

  ctx = context.context()
  handle = ctx._handle
  device = ctx.device_name
  # May be warmup GPU.
  ops.EagerTensor([[3.0]], context=handle, device=device)

  # float32
  dtype = dtypes.float32.as_datatype_enum
  three = [[3.0]]
  with timer(label("EagerTensor([[3.0]])"), iters=n) as iters:
    for _ in iters:
      ops.EagerTensor(three, context=handle, device=device, dtype=dtype)

  np_3 = np.array([[3.0]], dtype=np.float32)
  with timer(label("EagerTensor(np.array([[3.0]]))"), iters=n) as iters:
    for _ in iters:
      ops.EagerTensor(np_3, context=handle, device=device, dtype=dtype)

  # int32.
  # This is interesting since int32 will be kept on host memory for the GPU
  # case.
  dtype = dtypes.int32.as_datatype_enum
  three = [[3]]
  with timer(label("EagerTensor([[3]])"), iters=n) as iters:
    for _ in iters:
      ops.EagerTensor(three, context=handle, device=device, dtype=dtype)

  np_3 = np.array([[3]], dtype=np.int32)
  with timer(label("EagerTensor(np.array([[3]]))"), iters=n) as iters:
    for _ in iters:
      ops.EagerTensor(np_3, context=handle, device=device, dtype=dtype)

def benchmark_matmul(shape, n, use_gpu=False):
  """Benchmark for matrix multiplication using tf.matmul."""
  transpose_b = (shape[0] != shape[1])
  m = random_ops.random_uniform(shape)
  if use_gpu:
    m = m.gpu()
    # Warm up the GPU - the very first kernel invocation
    # seems to require a bunch of setup.
    math_ops.matmul(m, m, transpose_b=transpose_b)

  def label(s):
    return "MatMul {}: {:30s}".format(shape, s)

  if not use_gpu:
    a = m.cpu().numpy()
    b = a.T if transpose_b else a
    with timer(label("np.dot"), iters=n) as iters:
      for _ in iters:
        np.dot(a, b)

  with timer(label("tf.matmul"), iters=n) as iters:
    for _ in iters:
      math_ops.matmul(m, m, transpose_b=transpose_b)

  with timer(label("gen_math_ops.mat_mul"), iters=n) as iters:
    for _ in iters:
      gen_math_ops._mat_mul(m, m, transpose_b=transpose_b)

  inputs = [m, m]
  # pylint: disable=protected-access
  ctx_handle = context.context()._handle
  # pylint: enable=protected-access
  attrs = ("transpose_a", False, "transpose_b", transpose_b, "T",
           m.dtype.as_datatype_enum)
  with timer(label("TFE_Py_Execute"), iters=n) as iters:
    for _ in iters:
      pywrap_tensorflow.TFE_Py_Execute(ctx_handle, None, "MatMul",
                                       inputs, attrs, 1)

  f = function.defun(math_ops.matmul)
  with timer(label("defun(tf.matmul)"), iters=n) as iters:
    for _ in iters:
      f(m, m, transpose_b=transpose_b)

def benchmark_multiply(shape, n, use_gpu=False):
  m = random_ops.random_uniform(shape)
  if use_gpu:
    m = m.gpu()
    # Warm up the GPU - the very first kernel invocation
    # seems to require a bunch of setup.
    _ = m * m

  def label(s):
    return "Multiply {}: {:30s}".format(shape, s)

  if not use_gpu:
    a = m.cpu().numpy()
    with timer(label("np.multiply"), iters=n) as iters:
      for _ in iters:
        _ = a * a

  with timer(label("tf.multiply"), iters=n) as iters:
    for _ in iters:
      _ = m * m

class BenchmarksTest(test_util.TensorFlowTestCase):

  def testBenchmarks(self):
    # This isn't actually a test, but benchmarks packaged as a test
    # so that continuous integration runs catch any breakages.
    print(context.context())
    benchmark_create_tensor(FLAGS.iters or 30000)
    benchmark_matmul([2, 2], FLAGS.iters or 30000)
    benchmark_matmul([100, 28 * 28], FLAGS.iters or 1000)
    benchmark_multiply([2], FLAGS.iters or 30000)

    if context.context().num_gpus() > 0:
      print("---- RUNNING ON GPU NOW ----")
      with context.device("/device:GPU:0"):
        benchmark_create_tensor(FLAGS.iters or 30000)
      benchmark_matmul([2, 2], FLAGS.iters or 30000, use_gpu=True)
      benchmark_matmul([100, 28 * 28], FLAGS.iters or 1000, use_gpu=True)
      benchmark_multiply([2], FLAGS.iters or 30000, use_gpu=True)

if __name__ == "__main__":
  parser = argparse.ArgumentParser()
  # Default iterations to 1 to keep continuos integration test times low.
  parser.add_argument(
      "--iters",
      type=int,
      default=1,
      help="Number of iterators for each test. None or 0 for auto-selection")
  FLAGS, unparsed = parser.parse_known_args()
  sys.argv = [sys.argv[0]] + unparsed
  test.main()


# -*- coding: utf-8 -*-
import os

import interface.session_events as se

################################################################################
def use_GUI_windows(vip):
    """Let the windows pop up that contain all the vip widgets.
    """
    message_cwd     = "Current working directory:\n{0}\n".format(os.getcwd())
    message_welcome = "Welcome to the Virtual Instrument Panel!\n"
    vip.GUI_feedback([message_cwd, message_welcome])

    se.bn_open_GUI_feedback(vip)
    se.bn_open_plots_12(vip)

    vip.show()

    print "\n/(use_GUI_windows)\n"

def customize_paths(vip, DIR_PATH_data):
    """Use the vip's .set method to set the path relevant to the user interface.
    """
    ### this sets several line edits to initial sensible values
    FILE_PATH_session  = DIR_PATH_data+os.sep+"session_init.txt"
    FILE_PATH_notes    = "K:\\_Computing\\MeasurementSoftware\\VIP_notes.txt"
    FILE_PATH_waveform = "C:/Users/Public/Documents/Signadyne/Examples/Waveforms/Gaussian.csv"

    ### The format is: vip.set(SESSION_KEY, REPLACEMENT_DICTIONARY)
    ### Note that we could also save those settings to a .txt file and load it.
    vip.set('Results', {'DIR_PATH_results'   : DIR_PATH_data})
    vip.set('Session', {'FILE_PATH_session'  : FILE_PATH_session})
    vip.set('Options', {'FILE_PATH_notes'    : FILE_PATH_notes})
    vip.set('H3344_1', {'FILE_PATH_waveform' : FILE_PATH_waveform})

    for index in vip._sessions_local.keys():
        vip.set('Session', {'F_dict_index' : str(index)})     # why is this not setting the index??/
        se.bn_vip_to_list_session(vip)

    vip.set('Session', {'F_dict_index'       : 'default'})

    ### Unless it already exists, create a results data file directory.
    if not os.path.isdir(DIR_PATH_data):
        os.makedirs(DIR_PATH_data)
    ###Finally, make the Data folder the working directory for our session.
    os.chdir(DIR_PATH_data)

    print "\n/(customize_DIR_and_FILE_paths)\n"


from time import sleep
from threading import Event, Thread

from zmq import (
    Context,
    HWM,
    NOBLOCK,
    Poller,
    POLLIN,
    PUB,
    PAIR,
    SUB,
    SUBSCRIBE,
    )

shutdown = Event()

class KillThread(Exception):
    """Raised when we want threads to die"""

class Heartbeat(Thread):

    def __init__(self, context, *args, **kw):
        self.context = context
        self.pub = self.context.socket(PAIR)
        self.pub.bind("inproc://#1")
        super(Heartbeat, self).__init__(*args, **kw)

    def cleanup(self):
        self.pub.send("DIE")
        self.pub.close()

    def run(self):
        try:
            x = 0
            while not shutdown.is_set():
                self.pub.send("BEAT.FOO.* %d, %s" % (x + 1, self.name))
                x += 1
                sleep(1)
        finally:
            print "%s exiting..." % self.name
            self.cleanup()

class Stethoscope(Thread):

    def __init__(self, context, *args, **kw):
        self.context = context
        self.recv = self.context.socket(PAIR)
        self.recv.connect("inproc://#1")

        self.pub = self.context.socket(PUB)
        self.pub.connect('tcp://localhost:7003')
        self.pub.setsockopt(HWM, 1000)

        self.poller = Poller()
        self.poller.register(self.recv, POLLIN)
        super(Stethoscope, self).__init__(*args, **kw)

    def cleanup(self):
        self.recv.close()
        self.pub.close()

    def run(self):
        try:
            while not shutdown.is_set():
                socks = dict(self.poller.poll())
                if socks.get(self.recv) == POLLIN:
                    msg = self.recv.recv()
                    self.pub.send(msg, flags=NOBLOCK)
                    if msg == "DIE":
                        raise KillThread
        except KillThread:
            print "%s exiting..." % self.name
        finally:
            self.cleanup()

context = Context()

heart = Heartbeat(context, name="Heartbeat Thread")
stethoscope = Stethoscope(context, name="Stethoscope Thread")

for t in (heart, stethoscope):
    t.start()

while True:
    try:
        # call thread.join to keep some control in the main thread
        while (heart.is_alive() or
               stethoscope.is_alive()):
            heart.join(timeout=0.1)
            stethoscope.join(timeout=0.1)

    except KeyboardInterrupt:
        shutdown.set()
        while (heart.is_alive() or
               stethoscope.is_alive()):
            heart.join(timeout=0.1)
            stethoscope.join(timeout=0.1)

        context.term()
        break