#!/usr/bin/env python3
# md_lj_ll_module.py

#------------------------------------------------------------------------------------------------#
# This software was written in 2016/17                                                           #
# by Michael P. Allen <m.p.allen@warwick.ac.uk>/<m.p.allen@bristol.ac.uk>                        #
# and Dominic J. Tildesley <d.tildesley7@gmail.com> ("the authors"),                             #
# to accompany the book "Computer Simulation of Liquids", second edition, 2017 ("the text"),     #
# published by Oxford University Press ("the publishers").                                       #
#                                                                                #
# LICENCE                                                                                #
# Creative Commons CC0 Public Domain Dedication.                                                 #
# To the extent possible under law, the authors have dedicated all copyright and related         #
# and neighboring rights to this software to the PUBLIC domain worldwide.                        #
# This software is distributed without any warranty.                                             #
# You should have received a copy of the CC0 Public Domain Dedication along with this software.  #
# If not, see <http://creativecommons.org/publicdomain/zero/1.0/>.                               #
#                                                                                #
# DISCLAIMER                                                                                #
# The authors and publishers make no warranties about the software, and disclaim liability       #
# for all uses of the software, to the fullest extent permitted by applicable law.               #
# The authors and publishers do not recommend use of this software for any purpose.              #
# It is made freely available, solely to clarify points made in the text. When using or citing   #
# the software, you should not imply endorsement by the authors or publishers.                   #
#------------------------------------------------------------------------------------------------#

"""Force routine for MD simulation, LJ atoms, using neighbour lists."""

fast = True # Change this to replace NumPy force evaluation with slower Python

class PotentialType:
    """A composite variable for interactions."""

    def __init__(self, cut, pot, vir, lap, ovr):
        self.cut = cut # the potential energy cut (but not shifted) at r_cut
        self.pot = pot # the potential energy cut-and-shifted at r_cut
        self.vir = vir # the virial
        self.lap = lap # the Laplacian
        self.ovr = ovr # a flag indicating overlap (i.e. pot too high to use)

    def __add__(self, other):
        cut = self.cut +  other.cut
        pot = self.pot +  other.pot
        vir = self.vir +  other.vir
        lap = self.lap +  other.lap
        ovr = self.ovr or other.ovr

        return PotentialType(cut,pot,vir,lap,ovr)

def introduction():
    """Prints out introductory statements at start of run."""

    print('Lennard-Jones potential')
    print('Cut-and-shifted version for dynamics')
    print('Cut (but not shifted) version also calculated')
    print('Diameter, sigma = 1')
    print('Well depth, epsilon = 1')
    if fast:
        print('Fast NumPy force routine')
    else:
        print('Slow Python force routine')
    print('Uses neighbour lists')

def conclusion():
    """Prints out concluding statements at end of run."""

    print('Program ends')

def force ( box, r_cut, r ):
    """Takes in box, cutoff range, and coordinate array, and calculates forces and potentials etc."""

    import numpy as np
    from itertools import product
    import math

    # It is assumed that positions are in units where box = 1
    # Forces are calculated in units where sigma = 1 and epsilon = 1
    # Uses neighbour lists

    n = r.shape[0]

    # Set up vectors to half the cells in neighbourhood of 3x3x3 cells in cubic lattice
    # The cells are chosen so that if (d0,d1,d2) appears, then (-d0,-d1,-d2) does not.
    d = np.array ( [ [ 0, 0, 0], [ 1, 0, 0], [ 1, 1, 0], [-1, 1, 0],
                     [ 0, 1, 0], [ 0, 0, 1], [-1, 0, 1], [ 1, 0, 1], [-1,-1, 1],
                     [ 0,-1, 1], [ 1,-1, 1], [-1, 1, 1], [ 0, 1, 1], [ 1, 1, 1] ] )

    r = r - np.rint(r) # Ensure all atoms in periodic box

    sr2_ovr      = 1.77 # Overlap threshold (pot > 100)
    r_cut_box    = r_cut / box
    r_cut_box_sq = r_cut_box ** 2
    box_sq       = box ** 2

    # Calculate potential at cutoff
    sr2     = 1.0 / r_cut**2 # in sigma=1 units
    sr6     = sr2 ** 3
    sr12    = sr6 **2
    pot_cut = sr12 - sr6 # Without numerical factor 4

    # Initialize
    f     = np.zeros_like(r)
    total = PotentialType ( cut=0.0, pot=0.0, vir=0.0, lap=0.0, ovr=False )

    # Calculate cell index triplets
    sc = math.floor(box/r_cut)                          # Number of cells along box edge
    c  = np.floor((r+0.5)*sc).astype(np.int_)           # N*3 array of cell indices for all atoms
    assert np.all(c>=0) and np.all(c<sc), 'Index error' # Simplistic "guard" against roundoff

    if fast:

        # Build list of arrays, each array holding positions of atoms in a cell
        # At the same time, define a matching set of force arrays in each cell
        # i and j number the atoms in each cell; we do not refer explicitly to indices in r
        rc, fc = [], []                        # Initially empty lists of positions and forces
        for ci in product(range(sc),repeat=3): # Triple loop over cells
            mask = np.all(c==ci,axis=1)        # Mask identifies atoms in this cell
            rc.append(r[mask,:])               # Copy atom coordinates into array, add to list
            fc.append(np.zeros_like(rc[-1]))   # Zero corresponding forces, add to list

        for ci1, rci in enumerate(rc):            # Loop over i-cells, getting all atoms in each i-cell as an array
            ci = np.unravel_index(ci1,(sc,sc,sc)) # Get i-cell triple-indices
            if rci.size==0:                       # Handle empty cell
                continue

            for dj in d:                                              # Loop over neighbouring j-cells
                cj  = ci + dj                                         # Compute neighbour j-cell triple-indices
                cj1 = np.ravel_multi_index(cj,(sc,sc,sc),mode='wrap') # Convert j-cell to single-index
                rcj = rc[cj1]                                         # Get atoms in j-cell as an array
                if rcj.size==0:                                       # Handle empty cell
                    continue

                rij      = rci[:,np.newaxis,:]-rcj[np.newaxis,:,:] # Separation vectors for all i and j
                rij      = rij - np.rint(rij)                      # PBCs in box=1 units
                rij_sq   = np.sum(rij**2,axis=2)                   # Squared separations
                in_range = rij_sq < r_cut_box_sq                   # Set flags for within cutoff

                if ci1==cj1:
                    np.fill_diagonal(in_range,False) # Eliminate i==j when i-cell==j-cell
                    np.fill_diagonal(rij_sq,1.0)     # Avoid divide-by-zero below

                rij_sq = rij_sq * box_sq                         # Now in sigma=1 units
                rij    = rij * box                               # Now in sigma=1 units
                sr2    = np.where ( in_range, 1.0/rij_sq, 0.0 )  # (sigma/rij)**2, only if in range
                ovr    = sr2 > sr2_ovr                           # Overlap if too close
                sr6    = sr2 ** 3
                sr12   = sr6 ** 2
                cut    = sr12 - sr6                              # LJ potential (cut but not shifted)
                vir    = cut + sr12                              # LJ virial
                pot    = np.where ( in_range, cut-pot_cut, 0.0 ) # LJ potential (cut-and-shifted)
                lap    = ( 22.0*sr12 - 5.0*sr6 ) * sr2           # LJ Laplacian
                fij    = vir * sr2                               # LJ scalar part of forces
                fij    = rij * fij[:,:,np.newaxis]               # LJ pair forces

                if ci1==cj1: # Correct for double-counting ij and ji when i-cell==j-cell
                    fij = fij / 2
                    total = total + PotentialType ( cut=np.sum(cut)/2, pot=np.sum(pot)/2,
                                                    vir=np.sum(vir)/2, lap=np.sum(lap)/2, ovr=np.any(ovr) )
                else:
                    total = total + PotentialType ( cut=np.sum(cut), pot=np.sum(pot),
                                                    vir=np.sum(vir), lap=np.sum(lap), ovr=np.any(ovr) )

                fc[ci1][:,:] = fc[ci1][:,:] + np.sum(fij,axis=1) # Aggregate force on atoms in i-cell
                fc[cj1][:,:] = fc[cj1][:,:] - np.sum(fij,axis=0) # Aggregate force on atoms in j-cell

        # Copy forces from list of cell arrays to main force array
        for ci in product(range(sc),repeat=3):                          # Triple loop over cells
            mask      = np.all(c==ci,axis=1)                            # Mask identifies atoms in this cell
            ci1       = np.ravel_multi_index(ci,(sc,sc,sc),mode='wrap') # Single-index
            f[mask,:] = fc[ci1]                                         # Copy atom forces from correct cell

    else:

        # Build list of arrays, each array holding indices of atoms in a cell
        # ki and kj are atom indices in the r array; i and j number the atoms in each cell
        k_array = np.arange(n)                 # Atom indices 0..N-1
        kc = []                                # Initially empty list of indices
        for ci in product(range(sc),repeat=3): # Triple loop over cells
            mask = np.all(c==ci,axis=1)        # Mask identifies atoms in this cell
            kc.append(k_array[mask])           # Copy atom indices into array, add to list

        for ci1, kci in enumerate(kc): # Loop over i-cells, getting atom indices as an array
            ci = np.unravel_index(ci1,(sc,sc,sc)) # Get i-cell triple-indices

            for dj in d: # Loop over neighbouring j-cells
                cj  = ci + dj                                         # Compute neighbour j-cell triple-indices
                cj1 = np.ravel_multi_index(cj,(sc,sc,sc),mode='wrap') # Convert to single-index
                kcj = kc[cj1]                                         # Get indices of atoms in j-cell as an array

                for i, ki in enumerate(kci):     # Loop over individual atoms in i-cell
                    j0 = i+1 if cj1==ci1 else 0  # Only look upwards if i-cell==j-cell
                    if j0 >= kcj.size:           # Handles (redundantly) empty j-cell and the case
                        continue                 # where j-cell==i-cell and i is last atom

                    for kj in kcj[j0:]:          # Loop over individual atoms in j-cell
                        rij = r[ki,:]-r[kj,:]    # Separation vector
                        rij = rij - np.rint(rij) # Periodic boundary conditions in box=1 units
                        rij_sq = np.sum(rij**2)  # Squared separation

                        if rij_sq < r_cut_box_sq: # Check within cutoff
                            rij_sq  = rij_sq * box_sq # Now in sigma=1 units
                            rij     = rij * box       # Now in sigma=1 units
                            sr2     = 1.0 / rij_sq    # (sigma/rij)**2
                            ovr     = sr2 > sr2_ovr   # Overlap if too close
                            sr6     = sr2 ** 3
                            sr12    = sr6 ** 2
                            cut     = sr12 - sr6                    # LJ potential (cut but not shifted)
                            vir     = cut + sr12                    # LJ virial
                            pot     = cut - pot_cut                 # LJ potential (cut-and-shifted)
                            lap     = ( 22.0*sr12 - 5.0*sr6 ) * sr2 # LJ Laplacian
                            fij     = rij * vir * sr2               # LJ forces
                            total   = total + PotentialType ( cut=cut, pot=pot, vir=vir, lap=lap, ovr=ovr )
                            f[ki,:] = f[ki,:] + fij
                            f[kj,:] = f[kj,:] - fij

    # Multiply results by numerical factors
    f         = f         * 24.0       # 24*epsilon
    total.cut = total.cut * 4.0        # 4*epsilon
    total.pot = total.pot * 4.0        # 4*epsilon
    total.vir = total.vir * 24.0 / 3.0 # 24*epsilon and divide virial by 3
    total.lap = total.lap * 24.0 * 2.0 # 24*epsilon and factor 2 for ij and ji

    return total, f

def hessian ( box, r_cut, r, f ):
    """Calculates Hessian function (for 1/N correction to config temp)."""

    import numpy as np
    from itertools import product
    import math

    # This routine is only needed in a constant-energy ensemble
    # It is assumed that positions are in units where box = 1
    # but the result is given in units where sigma = 1 and epsilon = 1
    # It is assumed that forces have already been calculated in array f
    # Uses neighbour lists

    n = r.shape[0]
    assert np.all ( r.shape==f.shape ), 'Dimension mismatch in hessian'

    # Set up vectors to half the cells in neighbourhood of 3x3x3 cells in cubic lattice
    # The cells are chosen so that if (d1,d2,d3) appears, then (-d1,-d2,-d3) does not.
    d = np.array ( [ [ 0, 0, 0], [ 1, 0, 0], [ 1, 1, 0], [-1, 1, 0],
                     [ 0, 1, 0], [ 0, 0, 1], [-1, 0, 1], [ 1, 0, 1], [-1,-1, 1],
                     [ 0,-1, 1], [ 1,-1, 1], [-1, 1, 1], [ 0, 1, 1], [ 1, 1, 1] ] )

    r = r - np.rint(r) # Ensure all atoms in periodic box

    r_cut_box    = r_cut / box
    r_cut_box_sq = r_cut_box ** 2
    box_sq       = box ** 2

    hes = 0.0

    # Calculate cell index triplets
    sc = math.floor(box/r_cut)                          # Number of cells along box edge
    c  = np.floor((r+0.5)*sc).astype(np.int_)           # N*3 array of cell indices for all atoms
    assert np.all(c>=0) and np.all(c<sc), 'Index error' # Simplistic "guard" against roundoff

    if fast:

        # Build list of arrays, each array holding positions of atoms in a cell
        # At the same time, build a matching set of force arrays in each cell
        # i and j number the atoms in each cell; we do not refer explicitly to indices in r
        rc, fc = [], []                        # Initially empty lists of positions and forces
        for ci in product(range(sc),repeat=3): # Triple loop over cells
            mask = np.all(c==ci,axis=1)        # Mask identifies atoms in this cell
            rc.append(r[mask,:])               # Copy atom coordinates into array, add to list
            fc.append(f[mask,:])               # Copy corresponding forces, add to list

        for ci1, rci in enumerate(rc):            # Loop over i-cells, getting all atoms in each i-cell as an array
            ci = np.unravel_index(ci1,(sc,sc,sc)) # Get i-cell triple-indices
            fci = fc[ci1]                         # Get i-cell atom forces
            if rci.size==0:                       # Handle empty cell
                continue

            for dj in d:                                              # Loop over neighbouring j-cells
                cj = ci + dj                                          # Compute neighbour j-cell triple-indices
                cj1 = np.ravel_multi_index(cj,(sc,sc,sc),mode='wrap') # Convert j-cell to single-index
                rcj = rc[cj1]                                         # Get atoms in j-cell as an array
                fcj = fc[cj1]                                         # Get j-cell atom forces
                if rcj.size==0:                                       # Handle empty cell
                    continue

                rij      = rci[:,np.newaxis,:]-rcj[np.newaxis,:,:] # Separation vectors for all i and j
                rij      = rij - np.rint(rij)                      # PBCs in box=1 units
                rij_sq   = np.sum(rij**2,axis=2)                   # Squared separations
                in_range = rij_sq < r_cut_box_sq                   # Set flags for within cutoff

                if ci1==cj1:
                    np.fill_diagonal(in_range,False) # Eliminate i=j when i-cell is j-cell
                    np.fill_diagonal(rij_sq,1.0)     # Avoid divide-by-zero below

                rij_sq   = rij_sq * box_sq                         # Now in sigma=1 units
                rij      = rij * box                               # Now in sigma=1 units
                fij      = fci[:,np.newaxis,:]-fcj[np.newaxis,:,:] # Differences in forces for all i and j

                ff   = np.sum(fij*fij,axis=2)
                rf   = np.sum(rij*fij,axis=2)
                sr2  = np.where ( in_range, 1.0 / rij_sq, 0.0 ) # Only where in range
                sr6  = sr2 ** 3
                sr8  = sr6 * sr2
                sr10 = sr8 * sr2
                v1   = 24.0 * ( 1.0 - 2.0 * sr6 ) * sr8
                v2   = 96.0 * ( 7.0 * sr6 - 2.0 ) * sr10
                if ci1==cj1: # Correct for double-counting ij and ji
                    hes  = hes + np.sum(v1 * ff)/2 + np.sum(v2 * rf**2)/2
                else:
                    hes  = hes + np.sum(v1 * ff) + np.sum(v2 * rf**2)

    else:

        # Build list of arrays, each array holding indices of atoms in a cell
        # ki and kj are atom indices in the r array; i and j number the atoms in each cell
        k_array = np.arange(n)                 # Atom indices 0..N-1
        kc = []                                # Initially empty list of indices covering each cell
        for ci in product(range(sc),repeat=3): # Triple loop over cells
            mask = np.all(c==ci,axis=1)        # Mask identifies atoms in this cell
            kc.append(k_array[mask])           # Copy atom indices into array, add to list

        for ci1, kci in enumerate(kc): # Loop over i-cells, getting atom indices as an array
            ci = np.unravel_index(ci1,(sc,sc,sc)) # Get i-cell triple-indices

            for dj in d: # Loop over neighbouring j-cells
                cj  = ci + dj                                         # Compute neighbour j-cell triple-indices
                cj1 = np.ravel_multi_index(cj,(sc,sc,sc),mode='wrap') # Convert to single-index
                kcj = kc[cj1]                                         # Get indices of atoms in j-cell as an array

                for i, ki in enumerate(kci):    # Loop over individual atoms in i-cell
                    j0 = i+1 if cj1==ci1 else 0 # Only look upwards if i-cell==j-cell
                    if j0 >= kcj.size:          # Handles (redundantly) empty j-cell and the case
                        continue                # where j-cell==i-cell and i is last atom

                    for kj in kcj[j0:]:          # Loop over individual atoms in j-cell
                        rij = r[ki,:]-r[kj,:]    # Separation vector
                        rij = rij - np.rint(rij) # Periodic boundary conditions in box=1 units
                        rij_sq = np.sum(rij**2)  # Squared separation

                        if rij_sq < r_cut_box_sq:
                            rij_sq = rij_sq * box_sq # Now in sigma=1 units
                            rij = rij * box          # Now in sigma=1 units
                            fij = f[ki,:] - f[kj,:]  # Difference in forces

                            ff   = np.dot(fij,fij)
                            rf   = np.dot(rij,fij)
                            sr2  = 1.0 / rij_sq
                            sr6  = sr2 ** 3
                            sr8  = sr6 * sr2
                            sr10 = sr8 * sr2
                            v1   = 24.0 * ( 1.0 - 2.0 * sr6 ) * sr8
                            v2   = 96.0 * ( 7.0 * sr6 - 2.0 ) * sr10
                            hes  = hes + v1 * ff + v2 * rf**2

    return hes


# -*- coding: utf-8 -*-

# PLEASE DO NOT EDIT THIS FILE, IT IS GENERATED AND WILL BE OVERWRITTEN:
# https://github.com/ccxt/ccxt/blob/master/CONTRIBUTING.md#how-to-contribute-code

from ccxt.base.exchange import Exchange

# -----------------------------------------------------------------------------

try:
    basestring  # Python 3
except NameError:
    basestring = str  # Python 2
import hashlib
from ccxt.base.errors import ExchangeError
from ccxt.base.errors import AuthenticationError
from ccxt.base.errors import PermissionDenied
from ccxt.base.errors import ArgumentsRequired
from ccxt.base.errors import BadRequest
from ccxt.base.errors import InsufficientFunds
from ccxt.base.errors import InvalidOrder
from ccxt.base.errors import OrderNotFound
from ccxt.base.errors import NotSupported
from ccxt.base.errors import RateLimitExceeded
from ccxt.base.errors import ExchangeNotAvailable
from ccxt.base.errors import OnMaintenance
from ccxt.base.errors import InvalidNonce
from ccxt.base.decimal_to_precision import TICK_SIZE

class gemini(Exchange):

    def describe(self):
        return self.deep_extend(super(gemini, self).describe(), {
            'id': 'gemini',
            'name': 'Gemini',
            'countries': ['US'],
            # 600 requests a minute = 10 requests per second => 1000ms / 10 = 100ms between requests(private endpoints)
            # 120 requests a minute = 2 requests per second =>( 1000ms / rateLimit ) / 2 = 5(public endpoints)
            'rateLimit': 100,
            'version': 'v1',
            'has': {
                'CORS': None,
                'spot': True,
                'margin': False,
                'swap': False,
                'future': False,
                'option': False,
                'addMargin': False,
                'cancelOrder': True,
                'createDepositAddress': True,
                'createMarketOrder': None,
                'createOrder': True,
                'createReduceOnlyOrder': False,
                'fetchBalance': True,
                'fetchBidsAsks': None,
                'fetchBorrowRate': False,
                'fetchBorrowRateHistories': False,
                'fetchBorrowRateHistory': False,
                'fetchBorrowRates': False,
                'fetchBorrowRatesPerSymbol': False,
                'fetchClosedOrders': None,
                'fetchDepositAddress': None,  # TODO
                'fetchDepositAddressesByNetwork': True,
                'fetchDeposits': None,
                'fetchFundingHistory': False,
                'fetchFundingRate': False,
                'fetchFundingRateHistory': False,
                'fetchFundingRates': False,
                'fetchIndexOHLCV': False,
                'fetchIsolatedPositions': False,
                'fetchLeverage': False,
                'fetchLeverageTiers': False,
                'fetchMarkets': True,
                'fetchMarkOHLCV': False,
                'fetchMyTrades': True,
                'fetchOHLCV': True,
                'fetchOpenOrders': True,
                'fetchOrder': True,
                'fetchOrderBook': True,
                'fetchOrders': None,
                'fetchPosition': False,
                'fetchPositions': False,
                'fetchPositionsRisk': False,
                'fetchPremiumIndexOHLCV': False,
                'fetchTicker': True,
                'fetchTickers': True,
                'fetchTrades': True,
                'fetchTransactions': True,
                'fetchWithdrawals': None,
                'reduceMargin': False,
                'setLeverage': False,
                'setMarginMode': False,
                'setPositionMode': False,
                'withdraw': True,
            },
            'urls': {
                'logo': 'https://user-images.githubusercontent.com/1294454/27816857-ce7be644-6096-11e7-82d6-3c257263229c.jpg',
                'api': {
                    'public': 'https://api.gemini.com',
                    'private': 'https://api.gemini.com',
                    'web': 'https://docs.gemini.com',
                },
                'www': 'https://gemini.com/',
                'doc': [
                    'https://docs.gemini.com/rest-api',
                    'https://docs.sandbox.gemini.com',
                ],
                'test': {
                    'public': 'https://api.sandbox.gemini.com',
                    'private': 'https://api.sandbox.gemini.com',
                    # use the True doc instead of the sandbox doc
                    # since they differ in parsing
                    # https://github.com/ccxt/ccxt/issues/7874
                    # https://github.com/ccxt/ccxt/issues/7894
                    'web': 'https://docs.gemini.com',
                },
                'fees': [
                    'https://gemini.com/api-fee-schedule',
                    'https://gemini.com/trading-fees',
                    'https://gemini.com/transfer-fees',
                ],
            },
            'api': {
                'web': {
                    'get': [
                        'rest-api',
                    ],
                },
                'public': {
                    'get': {
                        'v1/symbols': 5,
                        'v1/symbols/details/{symbol}': 5,
                        'v1/pubticker/{symbol}': 5,
                        'v2/ticker/{symbol}': 5,
                        'v2/candles/{symbol}/{timeframe}': 5,
                        'v1/trades/{symbol}': 5,
                        'v1/auction/{symbol}': 5,
                        'v1/auction/{symbol}/history': 5,
                        'v1/pricefeed': 5,
                        'v1/book/{symbol}': 5,
                        'v1/earn/rates': 5,
                    },
                },
                'private': {
                    'post': {
                        'v1/order/new': 1,
                        'v1/order/cancel': 1,
                        'v1/wrap/{symbol}': 1,
                        'v1/order/cancel/session': 1,
                        'v1/order/cancel/all': 1,
                        'v1/order/status': 1,
                        'v1/orders': 1,
                        'v1/mytrades': 1,
                        'v1/notionalvolume': 1,
                        'v1/tradevolume': 1,
                        'v1/clearing/new': 1,
                        'v1/clearing/status': 1,
                        'v1/clearing/cancel': 1,
                        'v1/clearing/confirm': 1,
                        'v1/balances': 1,
                        'v1/notionalbalances/{currency}': 1,
                        'v1/transfers': 1,
                        'v1/addresses/{network}': 1,
                        'v1/deposit/{network}/newAddress': 1,
                        'v1/deposit/{currency}/newAddress': 1,
                        'v1/withdraw/{currency}': 1,
                        'v1/account/transfer/{currency}': 1,
                        'v1/payments/addbank': 1,
                        'v1/payments/methods': 1,
                        'v1/payments/sen/withdraw': 1,
                        'v1/balances/earn': 1,
                        'v1/earn/interest': 1,
                        'v1/approvedAddresses/{network}/request': 1,
                        'v1/approvedAddresses/account/{network}': 1,
                        'v1/approvedAddresses/{network}/remove': 1,
                        'v1/account': 1,
                        'v1/account/create': 1,
                        'v1/account/list': 1,
                        'v1/heartbeat': 1,
                    },
                },
            },
            'precisionMode': TICK_SIZE,
            'fees': {
                'trading': {
                    'taker': 0.0035,
                    'maker': 0.001,
                },
            },
            'httpExceptions': {
                '400': BadRequest,  # Auction not open or paused, ineligible timing, market not open, or the request was malformed, in the case of a private API request, missing or malformed Gemini private API authentication headers
                '403': PermissionDenied,  # The API key is missing the role necessary to access self private API endpoint
                '404': OrderNotFound,  # Unknown API entry point or Order not found
                '406': InsufficientFunds,  # Insufficient Funds
                '429': RateLimitExceeded,  # Rate Limiting was applied
                '500': ExchangeError,  # The server encountered an error
                '502': ExchangeNotAvailable,  # Technical issues are preventing the request from being satisfied
                '503': OnMaintenance,  # The exchange is down for maintenance
            },
            'timeframes': {
                '1m': '1m',
                '5m': '5m',
                '15m': '15m',
                '30m': '30m',
                '1h': '1hr',
                '6h': '6hr',
                '1d': '1day',
            },
            'exceptions': {
                'exact': {
                    'AuctionNotOpen': BadRequest,  # Failed to place an auction-only order because there is no current auction open for self symbol
                    'ClientOrderIdTooLong': BadRequest,  # The Client Order ID must be under 100 characters
                    'ClientOrderIdMustBeString': BadRequest,  # The Client Order ID must be a string
                    'ConflictingOptions': BadRequest,  # New orders using a combination of order execution options are not supported
                    'EndpointMismatch': BadRequest,  # The request was submitted to an endpoint different than the one in the payload
                    'EndpointNotFound': BadRequest,  # No endpoint was specified
                    'IneligibleTiming': BadRequest,  # Failed to place an auction order for the current auction on self symbol because the timing is not eligible, new orders may only be placed before the auction begins.
                    'InsufficientFunds': InsufficientFunds,  # The order was rejected because of insufficient funds
                    'InvalidJson': BadRequest,  # The JSON provided is invalid
                    'InvalidNonce': InvalidNonce,  # The nonce was not greater than the previously used nonce, or was not present
                    'InvalidOrderType': InvalidOrder,  # An unknown order type was provided
                    'InvalidPrice': InvalidOrder,  # For new orders, the price was invalid
                    'InvalidQuantity': InvalidOrder,  # A negative or otherwise invalid quantity was specified
                    'InvalidSide': InvalidOrder,  # For new orders, and invalid side was specified
                    'InvalidSignature': AuthenticationError,  # The signature did not match the expected signature
                    'InvalidSymbol': BadRequest,  # An invalid symbol was specified
                    'InvalidTimestampInPayload': BadRequest,  # The JSON payload contained a timestamp parameter with an unsupported value.
                    'Maintenance': OnMaintenance,  # The system is down for maintenance
                    'MarketNotOpen': InvalidOrder,  # The order was rejected because the market is not accepting new orders
                    'MissingApikeyHeader': AuthenticationError,  # The X-GEMINI-APIKEY header was missing
                    'MissingOrderField': InvalidOrder,  # A required order_id field was not specified
                    'MissingRole': AuthenticationError,  # The API key used to access self endpoint does not have the required role assigned to it
                    'MissingPayloadHeader': AuthenticationError,  # The X-GEMINI-PAYLOAD header was missing
                    'MissingSignatureHeader': AuthenticationError,  # The X-GEMINI-SIGNATURE header was missing
                    'NoSSL': AuthenticationError,  # You must use HTTPS to access the API
                    'OptionsMustBeArray': BadRequest,  # The options parameter must be an array.
                    'OrderNotFound': OrderNotFound,  # The order specified was not found
                    'RateLimit': RateLimitExceeded,  # Requests were made too frequently. See Rate Limits below.
                    'System': ExchangeError,  # We are experiencing technical issues
                    'UnsupportedOption': BadRequest,  # This order execution option is not supported.
                },
                'broad': {
                    'The Gemini Exchange is currently undergoing maintenance.': OnMaintenance,  # The Gemini Exchange is currently undergoing maintenance. Please check https://status.gemini.com/ for more information.
                    'We are investigating technical issues with the Gemini Exchange.': ExchangeNotAvailable,  # We are investigating technical issues with the Gemini Exchange. Please check https://status.gemini.com/ for more information.
                },
            },
            'options': {
                'fetchMarketsMethod': 'fetch_markets_from_web',
                'fetchTickerMethod': 'fetchTickerV1',  # fetchTickerV1, fetchTickerV2, fetchTickerV1AndV2
                'networkIds': {
                    'bitcoin': 'BTC',
                    'ethereum': 'ERC20',
                    'bitcoincash': 'BCH',
                    'litecoin': 'LTC',
                    'zcash': 'ZEC',
                    'filecoin': 'FIL',
                    'dogecoin': 'DOGE',
                    'tezos': 'XTZ',
                },
                'networks': {
                    'BTC': 'bitcoin',
                    'ERC20': 'ethereum',
                    'BCH': 'bitcoincash',
                    'LTC': 'litecoin',
                    'ZEC': 'zcash',
                    'FIL': 'filecoin',
                    'DOGE': 'dogecoin',
                    'XTZ': 'tezos',
                },
            },
        })

    def fetch_markets(self, params={}):
        method = self.safe_value(self.options, 'fetchMarketsMethod', 'fetch_markets_from_api')
        return getattr(self, method)(params)

    def fetch_markets_from_web(self, params={}):
        response = self.webGetRestApi(params)
        sections = response.split('
Symbols and minimums
')
        numSections = len(sections)
        error = self.id + ' the ' + self.name + ' API doc HTML markup has changed, breaking the parser of order limits and precision info for ' + self.name + ' markets.'
        if numSections != 2:
            raise NotSupported(error)
        tables = sections[1].split('tbody>')
        numTables = len(tables)
        if numTables < 2:
            raise NotSupported(error)
        rows = tables[1].split("\n
\n")  # eslint-disable-line quotes
        numRows = len(rows)
        if numRows < 2:
            raise NotSupported(error)
        result = []
        # skip the first element(empty string)
        for i in range(1, numRows):
            row = rows[i]
            cells = row.split("	\n")  # eslint-disable-line quotes
            numCells = len(cells)
            if numCells < 5:
                raise NotSupported(error)
            #     [
            #         'btcusd',  # currency
            #         '0.00001 BTC(1e-5)',  # min order size
            #         '0.00000001 BTC(1e-8)',  # tick size
            #         '0.01 USD',  # quote currency price increment
            #         '
'
            #     ]
            marketId = cells[0].replace('', '')
            # base = self.safe_currency_code(baseId)
            minAmountString = cells[1].replace('', '')
            minAmountParts = minAmountString.split(' ')
            minAmount = self.safe_number(minAmountParts, 0)
            amountPrecisionString = cells[2].replace('', '')
            amountPrecisionParts = amountPrecisionString.split(' ')
            idLength = len(marketId) - 0
            startingIndex = idLength - 3
            quoteId = marketId[startingIndex:idLength]
            quote = self.safe_currency_code(quoteId)
            pricePrecisionString = cells[3].replace('', '')
            pricePrecisionParts = pricePrecisionString.split(' ')
            baseId = marketId.replace(quoteId, '')
            base = self.safe_currency_code(baseId)
            result.append({
                'id': marketId,
                'symbol': base + '/' + quote,
                'base': base,
                'quote': quote,
                'settle': None,
                'baseId': baseId,
                'quoteId': quoteId,
                'settleId': None,
                'type': 'spot',
                'spot': True,
                'margin': False,
                'swap': False,
                'future': False,
                'option': False,
                'active': None,
                'contract': False,
                'linear': None,
                'inverse': None,
                'contractSize': None,
                'expiry': None,
                'expiryDatetime': None,
                'strike': None,
                'optionType': None,
                'precision': {
                    'amount': self.safe_number(amountPrecisionParts, 0),
                    'price': self.safe_number(pricePrecisionParts, 0),
                },
                'limits': {
                    'leverage': {
                        'min': None,
                        'max': None,
                    },
                    'amount': {
                        'min': minAmount,
                        'max': None,
                    },
                    'price': {
                        'min': None,
                        'max': None,
                    },
                    'cost': {
                        'min': None,
                        'max': None,
                    },
                },
                'info': row,
            })
        return result

    def fetch_markets_from_api(self, params={}):
        response = self.publicGetV1Symbols(params)
        result = []
        for i in range(0, len(response)):
            marketId = response[i]
            market = marketId
            idLength = len(marketId) - 0
            baseId = marketId[0:idLength - 3]
            quoteId = marketId[idLength - 3:idLength]
            base = self.safe_currency_code(baseId)
            quote = self.safe_currency_code(quoteId)
            result.append({
                'id': marketId,
                'symbol': base + '/' + quote,
                'base': base,
                'quote': quote,
                'settle': None,
                'baseId': baseId,
                'quoteId': quoteId,
                'settleId': None,
                'type': 'spot',
                'spot': True,
                'margin': False,
                'swap': False,
                'future': False,
                'option': False,
                'active': None,
                'contract': False,
                'linear': None,
                'inverse': None,
                'contractSize': None,
                'expiry': None,
                'expiryDatetime': None,
                'strike': None,
                'optionType': None,
                'precision': {
                    'price': None,
                    'amount': None,
                },
                'limits': {
                    'leverage': {
                        'min': None,
                        'max': None,
                    },
                    'amount': {
                        'min': None,
                        'max': None,
                    },
                    'price': {
                        'min': None,
                        'max': None,
                    },
                    'cost': {
                        'min': None,
                        'max': None,
                    },
                },
                'info': market,
            })
        return result

    def fetch_order_book(self, symbol, limit=None, params={}):
        self.load_markets()
        request = {
            'symbol': self.market_id(symbol),
        }
        if limit is not None:
            request['limit_bids'] = limit
            request['limit_asks'] = limit
        response = self.publicGetV1BookSymbol(self.extend(request, params))
        return self.parse_order_book(response, symbol, None, 'bids', 'asks', 'price', 'amount')

    def fetch_ticker_v1(self, symbol, params={}):
        self.load_markets()
        market = self.market(symbol)
        request = {
            'symbol': market['id'],
        }
        response = self.publicGetV1PubtickerSymbol(self.extend(request, params))
        #
        #     {
        #         "bid":"9117.95",
        #         "ask":"9117.96",
        #         "volume":{
        #             "BTC":"1615.46861748",
        #             "USD":"14727307.57545006088",
        #             "timestamp":1594982700000
        #         },
        #         "last":"9115.23"
        #     }
        #
        return self.parse_ticker(response, market)

    def fetch_ticker_v2(self, symbol, params={}):
        self.load_markets()
        market = self.market(symbol)
        request = {
            'symbol': market['id'],
        }
        response = self.publicGetV2TickerSymbol(self.extend(request, params))
        #
        #     {
        #         "symbol":"BTCUSD",
        #         "open":"9080.58",
        #         "high":"9184.53",
        #         "low":"9063.56",
        #         "close":"9116.08",
        #         # Hourly prices descending for past 24 hours
        #         "changes":["9117.33","9105.69","9106.23","9120.35","9098.57","9114.53","9113.55","9128.01","9113.63","9133.49","9133.49","9137.75","9126.73","9103.91","9119.33","9123.04","9124.44","9117.57","9114.22","9102.33","9076.67","9074.72","9074.97","9092.05"],
        #         "bid":"9115.86",
        #         "ask":"9115.87"
        #     }
        #
        return self.parse_ticker(response, market)

    def fetch_ticker_v1_and_v2(self, symbol, params={}):
        tickerA = self.fetch_ticker_v1(symbol, params)
        tickerB = self.fetch_ticker_v2(symbol, params)
        return self.deep_extend(tickerA, {
            'open': tickerB['open'],
            'high': tickerB['high'],
            'low': tickerB['low'],
            'change': tickerB['change'],
            'percentage': tickerB['percentage'],
            'average': tickerB['average'],
            'info': tickerB['info'],
        })

    def fetch_ticker(self, symbol, params={}):
        method = self.safe_value(self.options, 'fetchTickerMethod', 'fetchTickerV1')
        return getattr(self, method)(symbol, params)

    def parse_ticker(self, ticker, market=None):
        #
        # fetchTickers
        #
        #     {
        #         "pair": "BATUSD",
        #         "price": "0.20687",
        #         "percentChange24h": "0.0146"
        #     }
        #
        # fetchTickerV1
        #
        #     {
        #         "bid":"9117.95",
        #         "ask":"9117.96",
        #         "volume":{
        #             "BTC":"1615.46861748",
        #             "USD":"14727307.57545006088",
        #             "timestamp":1594982700000
        #         },
        #         "last":"9115.23"
        #     }
        #
        # fetchTickerV2
        #
        #     {
        #         "symbol":"BTCUSD",
        #         "open":"9080.58",
        #         "high":"9184.53",
        #         "low":"9063.56",
        #         "close":"9116.08",
        #         # Hourly prices descending for past 24 hours
        #         "changes":["9117.33","9105.69","9106.23","9120.35","9098.57","9114.53","9113.55","9128.01","9113.63","9133.49","9133.49","9137.75","9126.73","9103.91","9119.33","9123.04","9124.44","9117.57","9114.22","9102.33","9076.67","9074.72","9074.97","9092.05"],
        #         "bid":"9115.86",
        #         "ask":"9115.87"
        #     }
        #
        volume = self.safe_value(ticker, 'volume', {})
        timestamp = self.safe_integer(volume, 'timestamp')
        symbol = None
        marketId = self.safe_string_lower(ticker, 'pair')
        baseId = None
        quoteId = None
        base = None
        quote = None
        if marketId is not None:
            if marketId in self.markets_by_id:
                market = self.markets_by_id[marketId]
            else:
                idLength = len(marketId) - 0
                if idLength == 7:
                    baseId = marketId[0:4]
                    quoteId = marketId[4:7]
                else:
                    baseId = marketId[0:3]
                    quoteId = marketId[3:6]
                base = self.safe_currency_code(baseId)
                quote = self.safe_currency_code(quoteId)
                symbol = base + '/' + quote
        if (symbol is None) and (market is not None):
            symbol = market['symbol']
            baseId = market['baseId'].upper()
            quoteId = market['quoteId'].upper()
            base = market['base']
            quote = market['quote']
        price = self.safe_string(ticker, 'price')
        last = self.safe_string_2(ticker, 'last', 'close', price)
        percentage = self.safe_string(ticker, 'percentChange24h')
        open = self.safe_string(ticker, 'open')
        baseVolume = self.safe_string(volume, baseId)
        quoteVolume = self.safe_string(volume, quoteId)
        return self.safe_ticker({
            'symbol': symbol,
            'timestamp': timestamp,
            'datetime': self.iso8601(timestamp),
            'high': self.safe_string(ticker, 'high'),
            'low': self.safe_string(ticker, 'low'),
            'bid': self.safe_string(ticker, 'bid'),
            'bidVolume': None,
            'ask': self.safe_string(ticker, 'ask'),
            'askVolume': None,
            'vwap': None,
            'open': open,
            'close': last,
            'last': last,
            'previousClose': None,  # previous day close
            'change': None,
            'percentage': percentage,
            'average': None,
            'baseVolume': baseVolume,
            'quoteVolume': quoteVolume,
            'info': ticker,
        }, market, False)

    def fetch_tickers(self, symbols=None, params={}):
        self.load_markets()
        response = self.publicGetV1Pricefeed(params)
        #
        #     [
        #         {
        #             "pair": "BATUSD",
        #             "price": "0.20687",
        #             "percentChange24h": "0.0146"
        #         },
        #         {
        #             "pair": "LINKETH",
        #             "price": "0.018",
        #             "percentChange24h": "0.0000"
        #         },
        #     ]
        #
        return self.parse_tickers(response, symbols)

    def parse_trade(self, trade, market=None):
        #
        # public fetchTrades
        #
        #     {
        #         "timestamp":1601617445,
        #         "timestampms":1601617445144,
        #         "tid":14122489752,
        #         "price":"0.46476",
        #         "amount":"28.407209",
        #         "exchange":"gemini",
        #         "type":"buy"
        #     }
        #
        # private fetchTrades
        #
        #      {
        #          "price":"3900.00",
        #          "amount":"0.00996",
        #          "timestamp":1638891173,
        #          "timestampms":1638891173518,
        #          "type":"Sell",
        #          "aggressor":false,
        #          "fee_currency":"EUR",
        #          "fee_amount":"0.00",
        #          "tid":73621746145,
        #          "order_id":"73621746059",
        #          "exchange":"gemini",
        #          "is_auction_fill":false,
        #          "is_clearing_fill":false,
        #          "symbol":"ETHEUR",
        #          "client_order_id":"1638891171610"
        #      }
        #
        timestamp = self.safe_integer(trade, 'timestampms')
        id = self.safe_string(trade, 'tid')
        orderId = self.safe_string(trade, 'order_id')
        feeCurrencyId = self.safe_string(trade, 'fee_currency')
        feeCurrencyCode = self.safe_currency_code(feeCurrencyId)
        fee = {
            'cost': self.safe_string(trade, 'fee_amount'),
            'currency': feeCurrencyCode,
        }
        priceString = self.safe_string(trade, 'price')
        amountString = self.safe_string(trade, 'amount')
        side = self.safe_string_lower(trade, 'type')
        symbol = self.safe_symbol(None, market)
        return self.safe_trade({
            'id': id,
            'order': orderId,
            'info': trade,
            'timestamp': timestamp,
            'datetime': self.iso8601(timestamp),
            'symbol': symbol,
            'type': None,
            'side': side,
            'takerOrMaker': None,
            'price': priceString,
            'cost': None,
            'amount': amountString,
            'fee': fee,
        }, market)

    def fetch_trades(self, symbol, since=None, limit=None, params={}):
        self.load_markets()
        market = self.market(symbol)
        request = {
            'symbol': market['id'],
        }
        response = self.publicGetV1TradesSymbol(self.extend(request, params))
        #
        #     [
        #         {
        #             "timestamp":1601617445,
        #             "timestampms":1601617445144,
        #             "tid":14122489752,
        #             "price":"0.46476",
        #             "amount":"28.407209",
        #             "exchange":"gemini",
        #             "type":"buy"
        #         },
        #     ]
        #
        return self.parse_trades(response, market, since, limit)

    def parse_balance(self, response):
        result = {'info': response}
        for i in range(0, len(response)):
            balance = response[i]
            currencyId = self.safe_string(balance, 'currency')
            code = self.safe_currency_code(currencyId)
            account = self.account()
            account['free'] = self.safe_string(balance, 'available')
            account['total'] = self.safe_string(balance, 'amount')
            result[code] = account
        return self.safe_balance(result)

    def fetch_balance(self, params={}):
        self.load_markets()
        response = self.privatePostV1Balances(params)
        return self.parse_balance(response)

    def parse_order(self, order, market=None):
        timestamp = self.safe_integer(order, 'timestampms')
        amount = self.safe_string(order, 'original_amount')
        remaining = self.safe_string(order, 'remaining_amount')
        filled = self.safe_string(order, 'executed_amount')
        status = 'closed'
        if order['is_live']:
            status = 'open'
        if order['is_cancelled']:
            status = 'canceled'
        price = self.safe_string(order, 'price')
        average = self.safe_string(order, 'avg_execution_price')
        type = self.safe_string(order, 'type')
        if type == 'exchange limit':
            type = 'limit'
        elif type == 'market buy' or type == 'market sell':
            type = 'market'
        else:
            type = order['type']
        fee = None
        marketId = self.safe_string(order, 'symbol')
        symbol = self.safe_symbol(marketId, market)
        id = self.safe_string(order, 'order_id')
        side = self.safe_string_lower(order, 'side')
        clientOrderId = self.safe_string(order, 'client_order_id')
        return self.safe_order({
            'id': id,
            'clientOrderId': clientOrderId,
            'info': order,
            'timestamp': timestamp,
            'datetime': self.iso8601(timestamp),
            'lastTradeTimestamp': None,
            'status': status,
            'symbol': symbol,
            'type': type,
            'timeInForce': None,
            'postOnly': None,
            'side': side,
            'price': price,
            'stopPrice': None,
            'average': average,
            'cost': None,
            'amount': amount,
            'filled': filled,
            'remaining': remaining,
            'fee': fee,
            'trades': None,
        }, market)

    def fetch_order(self, id, symbol=None, params={}):
        self.load_markets()
        request = {
            'order_id': id,
        }
        response = self.privatePostV1OrderStatus(self.extend(request, params))
        return self.parse_order(response)

    def fetch_open_orders(self, symbol=None, since=None, limit=None, params={}):
        self.load_markets()
        response = self.privatePostV1Orders(params)
        market = None
        if symbol is not None:
            market = self.market(symbol)  # throws on non-existent symbol
        return self.parse_orders(response, market, since, limit)

    def create_order(self, symbol, type, side, amount, price=None, params={}):
        self.load_markets()
        if type == 'market':
            raise ExchangeError(self.id + ' allows limit orders only')
        nonce = self.nonce()
        amountString = self.amount_to_precision(symbol, amount)
        priceString = self.price_to_precision(symbol, price)
        request = {
            'client_order_id': str(nonce),
            'symbol': self.market_id(symbol),
            'amount': amountString,
            'price': priceString,
            'side': side,
            'type': 'exchange limit',  # gemini allows limit orders only
        }
        response = self.privatePostV1OrderNew(self.extend(request, params))
        return {
            'info': response,
            'id': response['order_id'],
        }

    def cancel_order(self, id, symbol=None, params={}):
        self.load_markets()
        request = {
            'order_id': id,
        }
        return self.privatePostV1OrderCancel(self.extend(request, params))

    def fetch_my_trades(self, symbol=None, since=None, limit=None, params={}):
        if symbol is None:
            raise ArgumentsRequired(self.id + ' fetchMyTrades() requires a symbol argument')
        self.load_markets()
        market = self.market(symbol)
        request = {
            'symbol': market['id'],
        }
        if limit is not None:
            request['limit_trades'] = limit
        if since is not None:
            request['timestamp'] = int(since / 1000)
        response = self.privatePostV1Mytrades(self.extend(request, params))
        return self.parse_trades(response, market, since, limit)

    def withdraw(self, code, amount, address, tag=None, params={}):
        tag, params = self.handle_withdraw_tag_and_params(tag, params)
        self.check_address(address)
        self.load_markets()
        currency = self.currency(code)
        request = {
            'currency': currency['id'],
            'amount': amount,
            'address': address,
        }
        response = self.privatePostV1WithdrawCurrency(self.extend(request, params))
        return {
            'info': response,
            'id': self.safe_string(response, 'txHash'),
        }

    def nonce(self):
        return self.milliseconds()

    def fetch_transactions(self, code=None, since=None, limit=None, params={}):
        self.load_markets()
        request = {}
        if limit is not None:
            request['limit_transfers'] = limit
        if since is not None:
            request['timestamp'] = since
        response = self.privatePostV1Transfers(self.extend(request, params))
        return self.parse_transactions(response)

    def parse_transaction(self, transaction, currency=None):
        timestamp = self.safe_integer(transaction, 'timestampms')
        currencyId = self.safe_string(transaction, 'currency')
        code = self.safe_currency_code(currencyId, currency)
        address = self.safe_string(transaction, 'destination')
        type = self.safe_string_lower(transaction, 'type')
        status = 'pending'
        # When deposits show as Advanced or Complete they are available for trading.
        if transaction['status']:
            status = 'ok'
        fee = None
        feeAmount = self.safe_number(transaction, 'feeAmount')
        if feeAmount is not None:
            fee = {
                'cost': feeAmount,
                'currency': code,
            }
        return {
            'info': transaction,
            'id': self.safe_string(transaction, 'eid'),
            'txid': self.safe_string(transaction, 'txHash'),
            'timestamp': timestamp,
            'datetime': self.iso8601(timestamp),
            'network': None,
            'address': address,
            'addressTo': None,
            'addressFrom': None,
            'tag': None,  # or is it defined?
            'tagTo': None,
            'tagFrom': None,
            'type': type,  # direction of the transaction,('deposit' | 'withdraw')
            'amount': self.safe_number(transaction, 'amount'),
            'currency': code,
            'status': status,
            'updated': None,
            'fee': fee,
        }

    def parse_deposit_address(self, depositAddress, currency=None):
        #
        #      {
        #          address: "0xed6494Fe7c1E56d1bd6136e89268C51E32d9708B",
        #          timestamp: "1636813923098",
        #          addressVersion: "eV1"                                         }
        #      }
        #
        address = self.safe_string(depositAddress, 'address')
        return {
            'currency': currency,
            'network': None,
            'address': address,
            'tag': None,
            'info': depositAddress,
        }

    def fetch_deposit_addresses_by_network(self, code, params={}):
        self.load_markets()
        network = self.safe_string(params, 'network')
        if network is None:
            raise ArgumentsRequired(self.id + 'fetchDepositAddressesByNetwork() requires a network parameter')
        params = self.omit(params, 'network')
        networks = self.safe_value(self.options, 'networks', {})
        networkId = self.safe_string(networks, network, network)
        networkIds = self.safe_value(self.options, 'networkIds', {})
        networkCode = self.safe_string(networkIds, networkId, network)
        request = {
            'network': networkId,
        }
        response = self.privatePostV1AddressesNetwork(self.extend(request, params))
        results = self.parse_deposit_addresses(response, [code], False, {'network': networkCode, 'currency': code})
        return self.group_by(results, 'network')

    def sign(self, path, api='public', method='GET', params={}, headers=None, body=None):
        url = '/' + self.implode_params(path, params)
        query = self.omit(params, self.extract_params(path))
        if api == 'private':
            self.check_required_credentials()
            apiKey = self.apiKey
            if apiKey.find('account') < 0:
                raise AuthenticationError(self.id + ' sign() requires an account-key, master-keys are not-supported')
            nonce = self.nonce()
            request = self.extend({
                'request': url,
                'nonce': nonce,
            }, query)
            payload = self.json(request)
            payload = self.string_to_base64(payload)
            signature = self.hmac(payload, self.encode(self.secret), hashlib.sha384)
            headers = {
                'Content-Type': 'text/plain',
                'X-GEMINI-APIKEY': self.apiKey,
                'X-GEMINI-PAYLOAD': self.decode(payload),
                'X-GEMINI-SIGNATURE': signature,
            }
        else:
            if query:
                url += '?' + self.urlencode(query)
        url = self.urls['api'][api] + url
        return {'url': url, 'method': method, 'body': body, 'headers': headers}

    def handle_errors(self, httpCode, reason, url, method, headers, body, response, requestHeaders, requestBody):
        if response is None:
            if isinstance(body, basestring):
                feedback = self.id + ' ' + body
                self.throw_broadly_matched_exception(self.exceptions['broad'], body, feedback)
            return  # fallback to default error handler
        #
        #     {
        #         "result": "error",
        #         "reason": "BadNonce",
        #         "message": "Out-of-sequence nonce <1234> precedes previously used nonce <2345>"
        #     }
        #
        result = self.safe_string(response, 'result')
        if result == 'error':
            reason = self.safe_string(response, 'reason')
            message = self.safe_string(response, 'message')
            feedback = self.id + ' ' + message
            self.throw_exactly_matched_exception(self.exceptions['exact'], reason, feedback)
            self.throw_exactly_matched_exception(self.exceptions['exact'], message, feedback)
            self.throw_broadly_matched_exception(self.exceptions['broad'], message, feedback)
            raise ExchangeError(feedback)  # unknown message

    def create_deposit_address(self, code, params={}):
        self.load_markets()
        currency = self.currency(code)
        request = {
            'currency': currency['id'],
        }
        response = self.privatePostV1DepositCurrencyNewAddress(self.extend(request, params))
        address = self.safe_string(response, 'address')
        self.check_address(address)
        return {
            'currency': code,
            'address': address,
            'tag': None,
            'info': response,
        }

    def fetch_ohlcv(self, symbol, timeframe='5m', since=None, limit=None, params={}):
        self.load_markets()
        market = self.market(symbol)
        request = {
            'timeframe': self.timeframes[timeframe],
            'symbol': market['id'],
        }
        response = self.publicGetV2CandlesSymbolTimeframe(self.extend(request, params))
        #
        #     [
        #         [1591515000000,0.02509,0.02509,0.02509,0.02509,0],
        #         [1591514700000,0.02503,0.02509,0.02503,0.02509,44.6405],
        #         [1591514400000,0.02503,0.02503,0.02503,0.02503,0],
        #     ]
        #
        return self.parse_ohlcvs(response, market, timeframe, since, limit)


import sympy as sym

# Computing with Dirichlet conditions: -u''=2 and sines
x, L = sym.symbols('x L')
e_Galerkin = x*(L-x) - 8*L**2*sym.pi**(-3)*sym.sin(sym.pi*x/L)
e_colloc = x*(L-x) - 2*L**2*sym.pi**(-2)*sym.sin(sym.pi*x/L)

# Verify max error for x=L/2
dedx_Galerkin = sym.diff(e_Galerkin, x)
print((dedx_Galerkin.subs(x, L/2)))
dedx_colloc = sym.diff(e_colloc, x)
print((dedx_colloc.subs(x, L/2)))

# Compute max error: x=L/2, evaluate numerical, and simplify
print(('Max error Galerkin/least.sq.:', \
      sym.simplify(e_Galerkin.subs(x, L/2).evalf(n=3))))
print(('Max error colloc.:', \
      sym.simplify(e_colloc.subs(x, L/2).evalf(n=3))))
import sys
#sys.exit(0)

# Computing with Neumann and Dirichlet conditions: -u''=2
x, C, D = sym.symbols('x C D')
i, j = sym.symbols('i j', integer=True)

integrand = (i+1)*(j+1)*(1-x)**(i+j)
A_ij = sym.integrate(integrand, (x, 0, 1))
A_ij = sym.simplify(A_ij)
print(A_ij)
psi_i = (1-x)**(i+1)
integrand = 2*psi_i - D*(i+1)*(1-x)**i
b_i = sym.integrate(integrand, (x, 0, 1)) - C*psi_i.subs(x, 0)
b_i = sym.factor(sym.simplify(b_i))
print(b_i)
print((sym.expand(2 - (2+i)*(D+C))))

# Solving model2 problem with f(x) and fe1D.py
from u_xx_f_sympy import model2, x, C, D, L
m = 2
u = model2(x**m, L, C, D)
print(u)
#u_exact = lambda x: D + C*(x-L) + (1./6)*(L**3 - x**3)
u_exact = sym.lambdify([x, C, D, L], u)

import numpy as np
from fe1D import finite_element1D_naive, mesh_uniform
# Override C, D and L with numeric values
C = 5
D = 2
L = 4

d = 1

vertices, cells, dof_map = mesh_uniform(
    N_e=2, d=d, Omega=[0,L], symbolic=False)
vertices[1] = 3
essbc = {}
essbc[dof_map[-1][-1]] = D

c, A, b, timing = finite_element1D_naive(
    vertices, cells, dof_map,
    essbc,
    ilhs=lambda e, phi, r, s, X, x, h:
    phi[1][r](X, h)*phi[1][s](X, h),
    irhs=lambda e, phi, r, X, x, h:
    x**m*phi[0][r](X),
    blhs=lambda e, phi, r, s, X, x, h: 0,
    brhs=lambda e, phi, r, X, x, h:
    -C*phi[0][r](-1) if e == 0 else 0,
    intrule='GaussLegendre',
    verbose=False,
    )

# Visualize
from fe1D import u_glob
x, u, nodes = u_glob(c, cells, vertices, dof_map)
u_e = u_exact(x, C, D, L)
print((u_exact(nodes, C, D, L) - c))  # difference at the nodes
import matplotlib.pyplot as plt
plt.plot(x, u, 'b-', x, u_e, 'r--')
plt.legend(['finite elements, d=%d' %d, 'exact'], loc='upper left')
plt.savefig('tmp.png'); plt.savefig('tmp.pdf')
plt.show()


# -*- coding: utf-8 -*-

"""
    This module exports functions to initialize the Flask application.
"""

import random
from typing import Callable, Dict

import flask
import flask_babel

import orchard.errors
import orchard.extensions
import orchard.system_status

def create_app(config: str = 'Development') -> flask.Flask:
    """
        Create and initialize the Flask application.

        :param config: The name of the configuration class, valid values are ``Development``
                       (default), ``Production``, and ``Testing``.
        :return: The initialized Flask application.
    """
    configuration_values = {'Development', 'Production', 'Testing'}
    if config in configuration_values:
        config = 'orchard.configuration.{config}'.format(config = config)
    else:  # pragma: no cover.
        config = 'orchard.configuration.Development'

    name = __name__.split('.')[0]
    app = flask.Flask(name, instance_relative_config = True)
    app.config.from_object(config)
    app.config.from_object('instance.Configuration')

    # Always use English as default language during testing.
    if app.testing:  # pragma: no branch.
        app.config['BABEL_DEFAULT_LOCALE'] = 'en'

    _configure_blueprints(app)
    _configure_context_processor(app)
    _configure_extensions(app)
    _configure_logging(app)
    _configure_request_handlers(app)

    return app

def _configure_blueprints(app: flask.Flask):
    """
        Register the blueprints.

        :param app: The application instance.
    """
    app.register_blueprint(orchard.errors.blueprint)
    app.register_blueprint(orchard.system_status.blueprint)

def _configure_context_processor(app: flask.Flask):
    """
        Set up the global context processors.

        :param app: The application instance.
    """

    @app.context_processor
    def inject_jinja2() -> Dict[str, Callable]:
        """
            Inject more functions into the scope of Jinja2 templates.

            :return: A dictionary
        """
        jinja2_functions = {
            'hasattr': hasattr,
            'random_int': random.randint
        }

        return jinja2_functions

def _configure_extensions(app: flask.Flask):
    """
        Register the extensions with the app and configure them as needed.

        :param app: The application instance.
    """
    orchard.extensions.babel.init_app(app)
    orchard.extensions.cache.init_app(app)

def _configure_logging(app: flask.Flask):  # pragma: no cover.
    """
        Set up a file and a mail logger, unless the app is being debugged or tested.

        :param app: The application instance.
    """
    if app.debug or app.testing:
        return

    # noinspection PyUnresolvedReferences
    import logging
    import logging.handlers
    import os

    # Set up the file logger.
    log_path = app.config['LOG_PATH']
    if not os.path.isdir(log_path):
        os.makedirs(log_path)

    log_file = os.path.join(log_path, '{file_name}.log'.format(file_name = app.name))
    log_format = '%(asctime)s %(levelname)s: %(message)s [in %(pathname)s:%(lineno)d]'
    file_handler = logging.handlers.RotatingFileHandler(log_file, 'a', 1 * 1024 * 1024, 10)
    file_handler.setLevel(logging.INFO)
    file_handler.setFormatter(logging.Formatter(log_format))
    app.logger.setLevel(logging.INFO)
    app.logger.addHandler(file_handler)
    app.logger.info('{name} Startup'.format(name = app.config['PROJECT_NAME']))

    # Set up the mail logger.
    if app.config.get('MAIL_SERVER', '') == '':
        return

    credentials = None
    if app.config['MAIL_USERNAME'] or app.config['MAIL_PASSWORD']:
        credentials = (app.config['MAIL_USERNAME'], app.config['MAIL_PASSWORD'])

    server = (app.config['MAIL_SERVER'], app.config['MAIL_PORT'])
    sender = app.config['MAIL_FROM']
    receivers = app.config['ADMINS']
    subject = '{name} Failure'.format(name = app.config['PROJECT_NAME'])
    secure = None
    if app.config['MAIL_SSL']:
        secure = ()
    mail_handler = logging.handlers.SMTPHandler(server, sender, receivers, subject, credentials,
                                                secure)
    mail_handler.setLevel(logging.ERROR)
    app.logger.addHandler(mail_handler)

def _configure_request_handlers(app: flask.Flask):
    """
        Set up the global before and after request handlers.

        :param app: The application instance.
    """

    @app.before_request
    def before_request():
        """
            Set up a few things before handling the actual request.
        """
        flask.g.locale = flask_babel.get_locale()
        flask.g.project_name = app.config['PROJECT_NAME']

        # Set a default title.
        flask.g.title = app.config['PROJECT_NAME']

    @app.after_request
    def after_request(response: flask.Response) -> flask.Response:
        """
            Modify the response after the request has been handled.

            :return: The modified response.
        """
        # http://www.gnuterrypratchett.com/
        response.headers.add("X-Clacks-Overhead", "GNU Terry Pratchett")

        return response


import functools
import re
from typing import Any, Dict, Optional, Tuple, Union
from urllib.parse import urlsplit

from django.apps import apps
from django.contrib.auth.models import AnonymousUser
from django.http import HttpRequest, JsonResponse
from django.utils import timezone
from rest_framework import authentication
from rest_framework.exceptions import AuthenticationFailed
from rest_framework.request import Request

class PersonalAPIKeyAuthentication(authentication.BaseAuthentication):
    """A way of authenticating with personal API keys.
    Only the first key candidate found in the request is tried, and the order is:
    1. Request Authorization header of type Bearer.
    2. Request body.
    3. Request query string.
    """

    keyword = "Bearer"

    @classmethod
    def find_key_with_source(
        cls,
        request: Union[HttpRequest, Request],
        request_data: Optional[Dict[str, Any]] = None,
        extra_data: Optional[Dict[str, Any]] = None,
    ) -> Optional[Tuple[str, str]]:
        """Try to find personal API key in request and return it along with where it was found."""
        if "HTTP_AUTHORIZATION" in request.META:
            authorization_match = re.match(fr"^{cls.keyword}\s+(\S.+)$", request.META["HTTP_AUTHORIZATION"])
            if authorization_match:
                return authorization_match.group(1).strip(), "Authorization header"
        data = request.data if request_data is None and isinstance(request, Request) else request_data

        if data and "personal_api_key" in data:
            return data["personal_api_key"], "body"
        if "personal_api_key" in request.GET:
            return request.GET["personal_api_key"], "query string"
        if extra_data and "personal_api_key" in extra_data:
            # compatibility with /capture endpoint
            return extra_data["personal_api_key"], "query string data"
        return None

    @classmethod
    def find_key(
        cls,
        request: Union[HttpRequest, Request],
        request_data: Optional[Dict[str, Any]] = None,
        extra_data: Optional[Dict[str, Any]] = None,
    ) -> Optional[str]:
        """Try to find personal API key in request and return it."""
        key_with_source = cls.find_key_with_source(request, request_data, extra_data)
        return key_with_source[0] if key_with_source is not None else None

    @classmethod
    def authenticate(cls, request: Union[HttpRequest, Request]) -> Optional[Tuple[Any, None]]:
        personal_api_key_with_source = cls.find_key_with_source(request)
        if not personal_api_key_with_source:
            return None
        personal_api_key, source = personal_api_key_with_source
        PersonalAPIKey = apps.get_model(app_label="posthog", model_name="PersonalAPIKey")
        try:
            personal_api_key_object = (
                PersonalAPIKey.objects.select_related("user").filter(user__is_active=True).get(value=personal_api_key)
            )
        except PersonalAPIKey.DoesNotExist:
            raise AuthenticationFailed(detail=f"Personal API key found in request {source} is invalid.")
        personal_api_key_object.last_used_at = timezone.now()
        personal_api_key_object.save()
        assert personal_api_key_object.user is not None
        return personal_api_key_object.user, None

    @classmethod
    def authenticate_header(cls, request) -> str:
        return cls.keyword

class TemporaryTokenAuthentication(authentication.BaseAuthentication):
    def authenticate(self, request: Request):
        # if the Origin is different, the only authentication method should be temporary_token
        # This happens when someone is trying to create actions from the editor on their own website
        if (
            request.headers.get("Origin")
            and urlsplit(request.headers["Origin"]).netloc not in urlsplit(request.build_absolute_uri("/")).netloc
        ):
            if not request.GET.get("temporary_token"):
                raise AuthenticationFailed(
                    detail="No temporary_token set. "
                    + "That means you're either trying to access this API from a different site, "
                    + "or it means your proxy isn't sending the correct headers. "
                    + "See https://posthog.com/docs/deployment/running-behind-proxy for more information."
                )
        if request.GET.get("temporary_token"):
            User = apps.get_model(app_label="posthog", model_name="User")
            user = User.objects.filter(temporary_token=request.GET.get("temporary_token"))
            if not user.exists():
                raise AuthenticationFailed(detail="User doesn't exist")
            return (user.first(), None)
        return None

class PublicTokenAuthentication(authentication.BaseAuthentication):
    def authenticate(self, request: Request):
        if request.GET.get("share_token") and request.parser_context and request.parser_context.get("kwargs"):
            Dashboard = apps.get_model(app_label="posthog", model_name="Dashboard")
            dashboard = Dashboard.objects.filter(
                share_token=request.GET.get("share_token"), pk=request.parser_context["kwargs"].get("pk"),
            ).first()
            if dashboard is None:
                raise AuthenticationFailed(detail="Dashboard doesn't exist")
            if dashboard.team.organization.for_internal_metrics:
                return None
            return (AnonymousUser(), None)
        return None

def authenticate_secondarily(endpoint):
    """
    DEPRECATED: Used for supporting legacy endpoints not on DRF.
    Authentication for function views.
    """

    @functools.wraps(endpoint)
    def wrapper(request: HttpRequest):
        if not request.user.is_authenticated:
            try:
                auth_result = PersonalAPIKeyAuthentication.authenticate(request)
                if isinstance(auth_result, tuple) and auth_result[0].__class__.__name__ == "User":
                    request.user = auth_result[0]
                else:
                    raise AuthenticationFailed("Authentication credentials were not provided.")
            except AuthenticationFailed as e:
                return JsonResponse({"detail": e.detail}, status=401)
        return endpoint(request)

    return wrapper


import attr
import logging
import os

from datetime import datetime
from feedparser import parse as parse_feed
from typing import List, Optional

from telegram_rss.config import FeedConfig
from telegram_rss.utils import save_as, get_default_directory, load_dict
from . import Entry, Channel, Feed

class FeedUpdater:
    def __init__(self, feed_config: FeedConfig, ext: str = ".json"):
        self.feed_config = feed_config
        self._feed: Optional[Feed] = None
        self._local_feed: Optional[Feed] = None
        self.local_file = os.path.join(
            get_default_directory(),
            "data",
            f"{self.feed_config.name}" + ext,
        )
        self.logger = logging.getLogger(feed_config.name)

    def __call__(self, save: bool = True) -> List[Entry]:
        return self.get_new_entries(save=save)

    def get_new_entries(self, save: bool = True) -> List[Entry]:
        entries: List[Entry] = list()
        if not self.feed or self.feed == self.local_feed:
            self.logger.info("No new feeds found")
            return entries
        for feed in self.feed:
            if feed in entries:
                continue
            if feed not in self.local_feed:
                entries.append(feed)
        if not entries:
            self.logger.debug("All feeds aleady in local_feeds")
            return entries
        if self.feed_config.only_today:
            self.logger.debug("Filter feeds published only today")
            now = datetime.now()
            for i in range(len(entries)):
                entry = entries[i]
                if not entry.time:
                    continue
                if entry.time.date() == now.date():
                    continue
                else:
                    del entries[i]
        self.logger.info(f"Found new {len(entries)} feeds")
        if entries and save:
            self.save_feed(self.feed)
            self.logger.debug(f"Saved {len(entries)} as {self.local_file}")
        return entries

    @property
    def feed(self) -> Feed:
        if self._feed:
            return self._feed
        if self.feed_config.save_bandwith:
            raw_feed = parse_feed(
                self.feed_config.source,
                etag=self.feed_config.etag,
                modified=self.feed_config.modified,
            )
        else:
            raw_feed = parse_feed(self.feed_config.source)
        if raw_feed.status == 304:
            return Feed()
        self.feed_config.etag = raw_feed.etag
        self.feed_config.modified = raw_feed.modified
        self._feed = Feed.from_feedparser(raw_feed)
        return self._feed

    @property
    def local_feed(self) -> Feed:
        if self._local_feed:
            return self._local_feed
        if not os.path.isfile(self.local_file):
            return Feed()
        feed_data = load_dict(self.local_file)
        self._local_feed = Feed(**feed_data)
        return self._local_feed

    def save_feed(self, feed: Feed):
        feed_data = attr.asdict(feed, recurse=True)
        save_as(feed_data, self.local_file)

    @property
    def channel(self) -> Optional[Channel]:
        return self.feed.channel or self.local_feed.channel


#Imports library
import socket

#Creates instance of 'Socket'
s = socket.socket()

hostname = 'tutorialspi' #Server IP/Hostname
port = 8000 #Server Port

s.connect((hostname,port)) #Connects to server

while True:
    x = raw_input("Enter message: ") #Gets the message to be sent
    s.send(x.encode()) #Encodes and sends message (x)


from cs50 import get_string
import re

def letters_counter(t, a):
    c = 0
    for i in t:
        if i in a or i in [j.upper() for j in a]:
            c += 1
    return c

def words_counter(t):
    match = re.split(" ", t)
    return len(match)

def sentences_counter(t):
    match = re.split("[.!?]", t)
    return len(match) - 1

def calculate(lc, wc, sc):
    l = (lc / wc) * 100
    s = (sc / wc) * 100
    index = 0.0588 * l - 0.296 * s - 15.8
    return round(index)

alphabet = ["a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z"]

text = get_string("Text: ")

letter_count = letters_counter(text, alphabet)
word_count = words_counter(text)
sentece_count = sentences_counter(text)

calc = calculate(letter_count, word_count, sentece_count)

if calc < 1:
    print("Before Grade 1")
elif calc >= 16:
    print("Grade 16+")
else:
    print(f"Grade {calc}")


import pytest

from .common import JSON, Cookies, Headers, Query, Resp, get_paths
from .test_plugin_falcon import api as falcon_api
from .test_plugin_flask import api as flask_api
from .test_plugin_flask_blueprint import api as flask_bp_api
from .test_plugin_flask_view import api as flask_view_api
from .test_plugin_starlette import api as starlette_api

@pytest.mark.parametrize(
    "api", [flask_api, flask_bp_api, flask_view_api, falcon_api, starlette_api]
)
def test_plugin_spec(api):
    models = {
        m.__name__: m.schema(ref_template="#/components/schemas/{model}")
        for m in (Query, JSON, Resp, Cookies, Headers)
    }
    for name, schema in models.items():
        assert api.spec["components"]["schemas"][name] == schema

    assert api.spec["tags"] == [{"name": tag} for tag in ("test", "health", "api")]

    assert get_paths(api.spec) == [
        "/api/user/{name}",
        "/api/user_annotated/{name}",
        "/ping",
    ]

    ping = api.spec["paths"]["/ping"]["get"]
    assert ping["tags"] == ["test", "health"]
    assert ping["parameters"][0]["in"] == "header"
    assert ping["summary"] == "summary"
    assert ping["description"] == "description"
    assert ping["operationId"] == "get_/ping"

    user = api.spec["paths"]["/api/user/{name}"]["post"]
    assert user["tags"] == ["api", "test"]
    assert (
        user["requestBody"]["content"]["application/json"]["schema"]["$ref"]
        == "#/components/schemas/JSON"
    )
    assert len(user["responses"]) == 3

    params = user["parameters"]
    for param in params:
        if param["in"] == "path":
            assert param["name"] == "name"
        elif param["in"] == "query":
            assert param["name"] == "order"


from sqlalchemy.orm import Session
from fastapi import Depends, Security
from starlette.requests import Request

from app.core.jwt import validate_token, reusable_oauth2
from app.api.utils.db import get_db

def get_current_user(request: Request, token: str = Security(reusable_oauth2)):
    print(token)
    return request.state.user

def get_validated_current_user(db: Session = Depends(get_db), token: str = Security(reusable_oauth2)):  # yapf: disable
    return validate_token(db, token)


from os.path import dirname, basename, isfile, join
import glob

modules = glob.glob(join(dirname(__file__), '*.py'))
__all__ = [basename(f)[:-3] for f in modules if isfile(f) and not f.endswith('__init__.py')]


import os
import argparse
import re

import utils

def aws_refresh_report(manifest, fname):
    """
    Generate a aws refresh report by looking into the log file
    generated during aws refresh script running. The output is a report
    containing the number of files is copied, total amount in GB was copied

    Args:
        manifest(tsv): GDC manifest (active or legacy)
        fname(str): the log file of running aws refresh script
    """
    try:
        with open(fname) as f:
            content = f.readlines()
    except IOError as e:
        print(e)
        os._exit(1)

    lines = [x.strip() for x in content]

    total_copying_files = 0
    total_data = 0
    awscli_copied_objects = set()
    awscli_copied_data = 0
    streaming_copied_objects = set()
    streaming_copied_data = 0

    for line in lines:
        pattern = "Total files need to be replicated: (.*)$"
        m = re.search(pattern, line)
        if m:
            total_copying_files = max(total_copying_files, int(m.group(1)))

        pattern = ".*aws s3 mv s3://.*/(.{36})/.*"
        m = re.search(pattern, line)
        if m:
            awscli_copied_objects.add(m.group(1))

        pattern = ".*aws s3 cp s3://gdcbackup/(.{36})/.*"
        m = re.search(pattern, line)
        if m:
            awscli_copied_objects.add(m.group(1))

        pattern = "successfully stream file (.{36})/.*"
        m = re.search(pattern, line)
        if m:
            streaming_copied_objects.add(m.group(1))

    files, headers = utils.get_fileinfo_list_from_csv_manifest(manifest)
    file_dict = {}
    for fi in files:
        file_dict[fi["id"]] = fi

    manifest_copied_files = 0
    for uuid in awscli_copied_objects:
        if uuid in file_dict:
            manifest_copied_files +=1
            awscli_copied_data += file_dict[uuid]["size"]*1.0/1024/1024/1024

    for uuid in streaming_copied_objects:
        if uuid in file_dict:
            manifest_copied_files +=1
            streaming_copied_data += file_dict[uuid]["size"]*1.0/1024/1024/1024

    report = """
    Number of files need to be copied {}. Total {} (GiB)
    Number of files were copied successfully via aws cli {}. Total {}(GiB)
    Number of files were copied successfully via gdc api {}. Total {}(GiB)
    """.format(
            total_copying_files,
            total_data,
            len(awscli_copied_objects),
            awscli_copied_data,
            len(streaming_copied_objects),
            streaming_copied_data
        )

    print(report)

    copied_files = []
    for uuid in awscli_copied_objects.union(streaming_copied_objects):
        if uuid in file_dict:
            copied_files.append(file_dict[uuid])

    print("Saving list of copied files")
    utils.write_csv(manifest.split("/")[-1][:-4] + "_aws_copied.tsv", copied_files, fieldnames=headers)
    return report

def aws_refresh_validate(fname):
    """
    Validate the aws data refresh by looking into the log after validation
    script finished.
    """
    try:
        with open(fname) as f:
            content = f.readlines()
    except IOError as e:
        print(e)
        print("Please run the dcf validation job first")
        os._exit(1)

    lines = [x.strip() for x in content]
    for line in lines:
        if "TOTAL AWS COPY FAILURE CASES" in line:
            print(line)
            return False
    return True

def parse_arguments():
    parser = argparse.ArgumentParser()
    subparsers = parser.add_subparsers(title="action", dest="action")

    aws_refresh_cmd = subparsers.add_parser("aws_refresh_report")
    aws_refresh_cmd.add_argument("--manifest", required=True)
    aws_refresh_cmd.add_argument("--log_file", required=True)

    aws_validate_cmd = subparsers.add_parser("aws_refresh_validate")
    aws_validate_cmd.add_argument("--manifest", required=True)
    aws_validate_cmd.add_argument("--log_file", required=True)

    return parser.parse_args()

def main():
    args = parse_arguments()
    fname = args.log_file
    manifest = args.manifest

    if args.action == "aws_refresh_report":
        aws_refresh_report(manifest, fname)
    elif args.action == "aws_refresh_validate":
        if aws_refresh_validate(fname):
            print("All the files in the manifest have been copied to aws dcf buckets")
        else:
            print("The manifest validation fails")

if __name__ == "__main__":
    main()


# coding=utf-8
# --------------------------------------------------------------------------
# Code generated by Microsoft (R) AutoRest Code Generator.
# Changes may cause incorrect behavior and will be lost if the code is
# regenerated.
# --------------------------------------------------------------------------

from msrest.serialization import Model

class MicrosoftgraphcalendarGroup(Model):
    """MicrosoftgraphcalendarGroup.

    :param id:
    :type id: str
    :param name:
    :type name: str
    :param class_id:
    :type class_id: str
    :param change_key:
    :type change_key: str
    :param calendars:
    :type calendars: list[~users.models.Microsoftgraphcalendar]
    """

    _attribute_map = {
        'id': {'key': 'id', 'type': 'str'},
        'name': {'key': 'name', 'type': 'str'},
        'class_id': {'key': 'classId', 'type': 'str'},
        'change_key': {'key': 'changeKey', 'type': 'str'},
        'calendars': {'key': 'calendars', 'type': '[Microsoftgraphcalendar]'},
    }

    def __init__(self, id=None, name=None, class_id=None, change_key=None, calendars=None):
        super(MicrosoftgraphcalendarGroup, self).__init__()
        self.id = id
        self.name = name
        self.class_id = class_id
        self.change_key = change_key
        self.calendars = calendars