import json
import os

from unittest import TestCase, mock
from climacell.api import Client, Measurement, Response, Error
from climacell.fields import (
    FIELD_TEMP, FIELD_DEW_POINT, FIELD_HUMIDITY,
    FIELD_WIND_SPEED, FIELD_WIND_GUST, FIELD_WIND_DIRECTION,
    FIELD_SUNRISE, FIELD_SUNSET,
)
from climacell.utils import join_fields

ERROR_FILE = os.path.dirname(__file__) + '/data/error_example.json'
HOURLY_FILE = os.path.dirname(__file__) + '/data/hourly_example.json'
DAILY_FILE = os.path.dirname(__file__) + '/data/daily_example.json'
NOWCAST_FILE = os.path.dirname(__file__) + '/data/nowcast_example.json'

class MockResponse:
    def __init__(self, data, status_code):
        self.data = data
        self.status_code = status_code

    def json(self):
        return self.data

def mock_requests_get(*args, **kwargs):
    base_url = 'https://api.climacell.co/v3'

    if args[0] is None:
        return MockResponse(None, 404)
    elif args[0] == base_url + '/weather/forecast/hourly':
        file = HOURLY_FILE
    elif args[0] == base_url + '/weather/nowcast':
        file = NOWCAST_FILE
    elif args[0] == base_url + '/weather/forecast/daily':
        file = DAILY_FILE

    with open(file) as json_file:
        data = json.load(json_file)

    return MockResponse(data, 200)

class TestMeasurement(TestCase):
    def test_measurement__str__(self):
        m = Measurement('temp', 13.04, 'C', '2021-01-14T21:00:00.000Z')
        self.assertEqual('temp: 13.04 C at 2021-01-14 21:00:00+00:00', str(m))

        m = Measurement('temp', 13.04, None, '2021-01-14T21:00:00.000Z')
        self.assertEqual('temp: 13.04 at 2021-01-14 21:00:00+00:00', str(m))

class TestResponse(TestCase):
    def test_get_measurements_error(self):
        with open(ERROR_FILE) as f:
            data = json.load(f)
        mock_response = MockResponse(data, 400)
        response = Response(mock_response, [FIELD_TEMP, FIELD_DEW_POINT, FIELD_HUMIDITY])

        self.assertTrue(response.has_error)
        error = response.get_measurements()
        self.assertTrue(isinstance(error, Error))

class TestError(TestCase):
    def test_error(self):
        with open(ERROR_FILE) as f:
            data = json.load(f)

        error = Error(data)

        self.assertEqual(400, error.status_code)
        self.assertEqual('Message body content not allowed.', error.message)
        self.assertEqual('BadRequest', error.code)

    def test_error_str(self):
        with open(ERROR_FILE) as f:
            data = json.load(f)

        error = Error(data)
        expected_str = 'BadRequest (400): Message body content not allowed.'
        self.assertEqual(expected_str, str(error))

class TestClient(TestCase):
    @mock.patch('climacell.api.requests.get', side_effect=mock_requests_get)
    def test_hourly(self, mock_get):
        client = Client('apikey')
        lat = 52.446023244274045
        lon = 4.819207798979252
        fields = [FIELD_TEMP, FIELD_DEW_POINT, FIELD_HUMIDITY]

        response = client.hourly(lat=lat, lon=lon, fields=fields)
        measurements = response.get_measurements()

        expected_params = {
            'lat': 52.446023244274045,
            'lon': 4.819207798979252,
            'start_time': 'now',
            'unit_system': 'si',
            'fields': join_fields([FIELD_TEMP, FIELD_DEW_POINT, FIELD_HUMIDITY]),
        }

        mock_get.assert_called_with(
            'https://api.climacell.co/v3/weather/forecast/hourly',
            params=expected_params,
            headers={'apikey': 'apikey'}
        )

        self.assertEqual(6, len(measurements))

    def test_hourly_invalid_start_time(self):
        client = Client('apikey')
        lat = 52.446023244274045
        lon = 4.819207798979252
        fields = [FIELD_TEMP, FIELD_DEW_POINT, FIELD_HUMIDITY]
        start_time = 'yesterday'

        self.assertRaises(
            ValueError, client.hourly, lat, lon, fields, start_time
        )

    def test_hourly_invalid_end_time(self):
        client = Client('apikey')
        lat = 52.446023244274045
        lon = 4.819207798979252
        fields = [FIELD_TEMP, FIELD_DEW_POINT, FIELD_HUMIDITY]
        start_time = 'now'
        end_time = 'tomorrow'

        self.assertRaises(
            ValueError, client.hourly, lat, lon, fields, start_time, end_time
        )

    @mock.patch('climacell.api.requests.get', side_effect=mock_requests_get)
    def test_hourly_valid_end_time(self, mock_get):
        client = Client('apikey')
        lat = 52.446023244274045
        lon = 4.819207798979252
        fields = [FIELD_TEMP, FIELD_DEW_POINT, FIELD_HUMIDITY]
        start_time = 'now'
        end_time = '2021-01-14T21:00:00.000Z'

        response = client.hourly(lat, lon, fields, start_time, end_time)
        self.assertFalse(response.has_error)

        expected_params = {
            'lat': 52.446023244274045,
            'lon': 4.819207798979252,
            'start_time': 'now',
            'unit_system': 'si',
            'fields': join_fields([FIELD_TEMP, FIELD_DEW_POINT, FIELD_HUMIDITY]),
            'end_time': '2021-01-14T21:00:00.000Z',
        }

        mock_get.assert_called_with(
            'https://api.climacell.co/v3/weather/forecast/hourly',
            params=expected_params,
            headers={'apikey': 'apikey'}
        )

    @mock.patch('climacell.api.requests.get', side_effect=mock_requests_get)
    def test_nowcast(self, mock_get):
        client = Client('apikey')
        lat = 52.446023244274045
        lon = 4.819207798979252
        timestep = 30
        fields = [
            FIELD_TEMP,
            FIELD_DEW_POINT,
            FIELD_HUMIDITY,
            FIELD_WIND_SPEED,
            FIELD_WIND_GUST,
            FIELD_WIND_DIRECTION,
            FIELD_SUNRISE,
            FIELD_SUNSET,
        ]

        response = client.nowcast(lat=lat, lon=lon, fields=fields, timestep=timestep)
        measurements = response.get_measurements()

        expected_params = {
            'lat': 52.446023244274045,
            'lon': 4.819207798979252,
            'timestep': 30,
            'start_time': 'now',
            'unit_system': 'si',
            'fields': join_fields(fields),
        }

        mock_get.assert_called_with(
            'https://api.climacell.co/v3/weather/nowcast',
            params=expected_params,
            headers={'apikey': 'apikey'}
        )
        # 13 timesteps, 8 measurements per timestep
        self.assertEqual(13 * 8, len(measurements))

    @mock.patch('climacell.api.requests.get', side_effect=mock_requests_get)
    def test_nowcast_valid_end_time(self, mock_get):
        client = Client('apikey')
        lat = 52.446023244274045
        lon = 4.819207798979252
        timestep = 30
        fields = [FIELD_TEMP]
        start_time = 'now'
        end_time = '2021-01-14T21:00:00.000Z'

        response = client.nowcast(
            lat=lat, lon=lon, fields=fields, timestep=timestep,
            start_time=start_time, end_time=end_time
        )

        self.assertFalse(response.has_error)

        expected_params = {
            'lat': 52.446023244274045,
            'lon': 4.819207798979252,
            'timestep': 30,
            'start_time': 'now',
            'unit_system': 'si',
            'fields': join_fields(fields),
            'end_time': end_time
        }

        mock_get.assert_called_with(
            'https://api.climacell.co/v3/weather/nowcast',
            params=expected_params,
            headers={'apikey': 'apikey'}
        )

    def test_nowcast_invalid_start_time(self):
        client = Client('apikey')
        lat = 52.446023244274045
        lon = 4.819207798979252
        timestep = 30
        fields = [FIELD_TEMP]
        start_time = 'yesterday'

        self.assertRaises(
            ValueError, client.nowcast, lat, lon, fields, timestep, start_time
        )

    def test_nowcast_invalid_end_time(self):
        client = Client('apikey')
        lat = 52.446023244274045
        lon = 4.819207798979252
        timestep = 30
        fields = [FIELD_TEMP]
        start_time = 'now'
        end_time = 'tomorrow'

        self.assertRaises(
            ValueError, client.nowcast, lat, lon, fields,
            timestep, start_time, end_time
        )

    @mock.patch('climacell.api.requests.get', side_effect=mock_requests_get)
    def test_daily(self, mock_get):
        client = Client('apikey')
        lat = 52.446023244274045
        lon = 4.819207798979252
        fields = [FIELD_TEMP, FIELD_DEW_POINT, FIELD_HUMIDITY]

        response = client.daily(lat=lat, lon=lon, fields=fields)
        measurements = response.get_measurements()

        expected_params = {
            'lat': 52.446023244274045,
            'lon': 4.819207798979252,
            'start_time': 'now',
            'unit_system': 'si',
            'fields': join_fields([FIELD_TEMP, FIELD_DEW_POINT, FIELD_HUMIDITY]),
        }

        mock_get.assert_called_with(
            'https://api.climacell.co/v3/weather/forecast/daily',
            params=expected_params,
            headers={'apikey': 'apikey'}
        )

        self.assertEqual(6, len(measurements))


import re

from django.db.models import query, Min, F
from django.views.generic import DetailView, ListView
from django.views.generic.edit import UpdateView
from django.urls import reverse
from django.shortcuts import redirect
from django.utils.http import is_safe_url
from django.http import QueryDict
from django.conf import settings

from .views import get_font
from .models import Notes, Source, Target
from .filters import NotesFilter, SourceFilter

class SourceListView(ListView):
    model = Source
    context_object_name = 'source'
    template_name = 'lexicon/list_source.html'
    paginate_by = 100

    def get_queryset(self):
        qs = super().get_queryset() # .values('id', 'token', 'lemma', 'morph', 'strongs_no_prefix', 'book', 'chapter', 'verse', 'notes')
        filtered = SourceFilter(self.request.GET, queryset=qs)
        return filtered.qs.distinct()[:100]

    def get_context_data(self, *args, **kwargs):
        context = super().get_context_data(*args, **kwargs)
        context['filter'] = SourceFilter(self.request.GET)
        return context

    def get_context_data(self, *args, **kwargs):
        context = super().get_context_data(*args, **kwargs)
        context['filter'] = SourceFilter(self.request.GET)

        query_dict = QueryDict(mutable=True)
        for key,val in self.request.GET.items():
            # removing page here, might have to add pagination to this view in its own right
            if val and key != 'page':
                query_dict[key] = val

        context['query_dict'] = query_dict
        context['base_page'] = reverse('navigate_source') + '?' + query_dict.urlencode()

        return context

class NavigateSource(SourceListView):
    paginate_by = 1
    template_name = 'lexicon/navigate_source.html'

    def get_context_data(self, *args, **kwargs):
        context = super().get_context_data(*args, **kwargs)
        qs = self.get_queryset()
        # (<django.core.paginator.Paginator object at 0x7f1a605f99a0>, <Page 2 of 4>, <QuerySet [ix79: figs-euphemism]>, True)
        paginator = self.paginate_queryset(qs, self.get_paginate_by(qs))

        try:
            # gets the first element of the queryset of the selected page
            occurrence = paginator[2].first()
        except:
            # a fallback to show something, it will display the wrong text though (but the right note)
            occurrence = qs.first()

        context['occurrence'] = occurrence
        # context['source'] = Source.objects.filter(book=occurrence['book'], chapter=occurrence['chapter'], verse=occurrence['verse'])
        context['source'] = Source.objects.filter(book=occurrence.book, chapter=occurrence.chapter, verse=occurrence.verse)
        context['target'] = Target.objects.filter(book=occurrence.book, chapter=occurrence.chapter, verse=occurrence.verse)
        book_nr = int(occurrence.book.split('-')[0])
        if book_nr > 40:
            font = 'gk'
        else:
            font = 'hb'
        context['font'] = font

        # make sure you pass the GET parameters along
        query_dict = QueryDict(mutable=True)
        for key,val in self.request.GET.items():
            if val and key != 'page':
                query_dict[key] = val

        context['query_dict'] = query_dict
        context['url'] = reverse('navigate_source') + '?' + query_dict.urlencode()
        context['base_page'] = reverse('list_source') + '?' + query_dict.urlencode()

        # prepare some nagivation
        page = paginator[1]
        if page.has_previous():
            context['previous_page'] = page.previous_page_number()
        if page.has_next():
            context['next_page'] = page.next_page_number()

        return context

class NotesListView(ListView):
    model = Notes
    context_object_name = 'notes'
    template_name = 'lexicon/list_notes.html'
    paginate_by = 100

    def get_queryset(self):
        qs = super().get_queryset().annotate(min_source=Min('source__id')).distinct().order_by(F('min_source').asc(nulls_last=True))
        filtered = NotesFilter(self.request.GET, queryset=qs)
        return filtered.qs.distinct()

    def get_context_data(self, *args, **kwargs):
        context = super().get_context_data(*args, **kwargs)
        context['filter'] = NotesFilter(self.request.GET)

        query_dict = QueryDict(mutable=True)
        for key,val in self.request.GET.items():
            # removing page here, might have to add pagination to this view in its own right
            if val and key != 'page':
                query_dict[key] = val

        context['base_page'] = reverse('navigate_notes') + '?' + query_dict.urlencode()

        return context

class NavigateNotes(NotesListView):
    paginate_by = 1
    template_name = 'lexicon/navigate_notes.html'

    def get_context_data(self, *args, **kwargs):
        context = super().get_context_data(*args, **kwargs)

        '''
        Here's the magic:

        Although this is a LIST View it really is used to only display a SINGLE object
        because it is paginated by 1.

        This is done so the entire queryset can be filtered by the user and the user can then
        go through each item in said queryset.

        This means that we need to add the actual information based on the paginated queryset here
        and not just the basic queryset.
        '''
        qs = self.get_queryset()
        # (<django.core.paginator.Paginator object at 0x7f1a605f99a0>, <Page 2 of 4>, <QuerySet [ix79: figs-euphemism]>, True)
        paginator = self.paginate_queryset(qs, self.get_paginate_by(qs))

        try:
            # gets the first element of the queryset of the selected page
            note = paginator[2].first()
        except:
            # a fallback to show something, it will display the wrong text though (but the right note)
            note = qs.first()

        context['source'] = Source.objects.filter(book=note.book, chapter=note.chapter, verse=note.verse)
        book_nr = int(note.book.split('-')[0])
        if book_nr > 40:
            font = 'gk'
        else:
            font = 'hb'
        context['font'] = font
        context['source'].first().strongs_no_prefix
        context['target'] = Target.objects.filter(book=note.book, chapter=note.chapter, verse=note.verse)

        # make sure you pass the GET parameters along
        query_dict = QueryDict(mutable=True)
        for key,val in self.request.GET.items():
            if val and key != 'page':
                query_dict[key] = val

        context['url'] = reverse('navigate_notes') + '?' + query_dict.urlencode()
        context['base_page'] = reverse('list_notes') + '?' + query_dict.urlencode()

        # prepare some nagivation
        page = paginator[1]
        if page.has_previous():
            context['previous_page'] = page.previous_page_number()
        if page.has_next():
            context['next_page'] = page.next_page_number()

        return context

class NotesDetailView(DetailView):
    model = Notes
    pk_url_kwarg = 'index'
    context_object_name = 'note'

    def get_context_data(self, *args, **kwargs):
        context = super().get_context_data(*args, **kwargs)
        note = self.object
        context['source'] = Source.objects.filter(book=note.book, chapter=note.chapter, verse=note.verse)
        book_nr = int(note.book.split('-')[0])
        if book_nr > 40:
            font = 'gk'
        else:
            font = 'hb'
        context['font'] = font
        context['source'].first().strongs_no_prefix
        context['target'] = Target.objects.filter(book=note.book, chapter=note.chapter, verse=note.verse)
        context['previous_note'] = Notes.objects.filter(index__lt=note.index).order_by('-index').first()
        context['next_note'] = Notes.objects.filter(index__gt=note.index).order_by('index').first()

        return context

class NotesUpdateView(UpdateView):
    model = Notes
    pk_url_kwarg = 'index'
    context_object_name = 'note'
    fields = 'supportreference annotation sourceword sourcewordoccurrence'.split()

    def get_success_url(self):
        next_url = self.request.GET.get('next', None)
        # do not accept any url
        if is_safe_url(next_url, allowed_hosts=settings.ALLOWED_HOSTS):
            return next_url


numbers = list()
while True:
    num = int(input('Insert a number: '))
    numbers.append(num)
    cont = str(input('Do you want to continue? [y/n]: ')).lower().strip()[0]
    while cont not in 'yn':
        cont = str(input('Do you want to continue? [y/n]: ')).lower().strip()[0]
    if cont == 'n':
        break
print(f'You inserted a total of {len(numbers)} numbers.')
print(f'The numbers, in descending order, are: {sorted(numbers, reverse=True)}.')
if 5 in numbers:
    print(f'The number 5 appear {numbers.count(5)} times.')
else:
    print('The number 5 don\'t appear in the list.')


# Copyright 2018 The Cirq Developers
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import itertools
import collections
from typing import Any, Iterable, cast, DefaultDict, TYPE_CHECKING, FrozenSet
from numpy import sqrt
from cirq import devices, ops, circuits, value
from cirq.devices.grid_qubit import GridQubit
from cirq.ops import raw_types
from cirq.value import Duration
from cirq.neutral_atoms import convert_to_neutral_atom_gates

if TYPE_CHECKING:
    import cirq

def _subgate_if_parallel_gate(gate: 'cirq.Gate') -> 'cirq.Gate':
    """Returns gate.sub_gate if gate is a ParallelGate, else returns gate"""
    return gate.sub_gate if isinstance(gate, ops.ParallelGate) else gate

def neutral_atom_gateset(max_parallel_z=None, max_parallel_xy=None):
    return ops.Gateset(
        ops.AnyIntegerPowerGateFamily(ops.CNotPowGate),
        ops.AnyIntegerPowerGateFamily(ops.CCNotPowGate),
        ops.AnyIntegerPowerGateFamily(ops.CZPowGate),
        ops.AnyIntegerPowerGateFamily(ops.CCZPowGate),
        ops.ParallelGateFamily(ops.ZPowGate, max_parallel_allowed=max_parallel_z),
        ops.ParallelGateFamily(ops.XPowGate, max_parallel_allowed=max_parallel_xy),
        ops.ParallelGateFamily(ops.YPowGate, max_parallel_allowed=max_parallel_xy),
        ops.ParallelGateFamily(ops.PhasedXPowGate, max_parallel_allowed=max_parallel_xy),
        ops.MeasurementGate,
        ops.IdentityGate,
        unroll_circuit_op=False,
        accept_global_phase_op=False,
    )

@value.value_equality
class NeutralAtomDevice(devices.Device):
    """A device with qubits placed on a grid."""

    def __init__(
        self,
        measurement_duration: 'cirq.DURATION_LIKE',
        gate_duration: 'cirq.DURATION_LIKE',
        control_radius: float,
        max_parallel_z: int,
        max_parallel_xy: int,
        max_parallel_c: int,
        qubits: Iterable[GridQubit],
    ) -> None:
        """Initializes the description of the AQuA device.

        Args:
            measurement_duration: the maximum duration of a measurement.
            gate_duration: the maximum duration of a gate
            control_radius: the maximum distance between qubits for a controlled
                gate. Distance is measured in units of the indices passed into
                the GridQubit constructor.
            max_parallel_z: The maximum number of qubits that can be acted on
                in parallel by a Z gate
            max_parallel_xy: The maximum number of qubits that can be acted on
                in parallel by a local XY gate
            max_parallel_c: the maximum number of qubits that can be acted on in
                parallel by a controlled gate. Must be less than or equal to the
                lesser of max_parallel_z and max_parallel_xy
            qubits: Qubits on the device, identified by their x, y location.
                Must be of type GridQubit

        Raises:
            ValueError: if the wrong qubit type is provided or if invalid
                parallel parameters are provided
        """
        self._measurement_duration = Duration(measurement_duration)
        self._gate_duration = Duration(gate_duration)
        self._control_radius = control_radius
        self._max_parallel_z = max_parallel_z
        self._max_parallel_xy = max_parallel_xy
        if max_parallel_c > min(max_parallel_z, max_parallel_xy):
            raise ValueError(
                "max_parallel_c must be less than or equal to the"
                "min of max_parallel_z and max_parallel_xy"
            )
        self._max_parallel_c = max_parallel_c
        self.xy_gateset_all_allowed = ops.Gateset(
            ops.ParallelGateFamily(ops.XPowGate),
            ops.ParallelGateFamily(ops.YPowGate),
            ops.ParallelGateFamily(ops.PhasedXPowGate),
            unroll_circuit_op=False,
            accept_global_phase_op=False,
        )
        self.controlled_gateset = ops.Gateset(
            ops.AnyIntegerPowerGateFamily(ops.CNotPowGate),
            ops.AnyIntegerPowerGateFamily(ops.CCNotPowGate),
            ops.AnyIntegerPowerGateFamily(ops.CZPowGate),
            ops.AnyIntegerPowerGateFamily(ops.CCZPowGate),
            unroll_circuit_op=False,
            accept_global_phase_op=False,
        )
        self.gateset = neutral_atom_gateset(max_parallel_z, max_parallel_xy)
        for q in qubits:
            if not isinstance(q, GridQubit):
                raise ValueError(f'Unsupported qubit type: {q!r}')
        self.qubits = frozenset(qubits)

    def qubit_set(self) -> FrozenSet['cirq.GridQubit']:
        return self.qubits

    def qubit_list(self):
        return [qubit for qubit in self.qubits]

    def decompose_operation(self, operation: ops.Operation) -> ops.OP_TREE:
        return convert_to_neutral_atom_gates.ConvertToNeutralAtomGates().convert(operation)

    def duration_of(self, operation: ops.Operation):
        """Provides the duration of the given operation on this device.

        Args:
            operation: the operation to get the duration of

        Returns:
            The duration of the given operation on this device

        Raises:
            ValueError: If the operation provided doesn't correspond to a native
                gate
        """
        self.validate_operation(operation)
        if isinstance(operation, (ops.GateOperation, ops.ParallelGateOperation)):
            if isinstance(operation.gate, ops.MeasurementGate):
                return self._measurement_duration
        return self._gate_duration

    def validate_gate(self, gate: ops.Gate):
        """Raises an error if the provided gate isn't part of the native gate set.

        Args:
            gate: the gate to validate

        Raises:
            ValueError: If the given gate is not part of the native gate set.
        """
        if gate not in self.gateset:
            if isinstance(gate, (ops.CNotPowGate, ops.CZPowGate, ops.CCXPowGate, ops.CCZPowGate)):
                raise ValueError('controlled gates must have integer exponents')
            raise ValueError(f'Unsupported gate: {gate!r}')

    def validate_operation(self, operation: ops.Operation):
        """Raises an error if the given operation is invalid on this device.

        Args:
            operation: the operation to validate

        Raises:
            ValueError: If the operation is not valid
        """
        if not isinstance(operation, (ops.GateOperation, ops.ParallelGateOperation)):
            raise ValueError(f'Unsupported operation: {operation!r}')

        # All qubits the operation acts on must be on the device
        for q in operation.qubits:
            if q not in self.qubits:
                raise ValueError(f'Qubit not on device: {q!r}')

        if operation not in self.gateset and not (
            operation in self.xy_gateset_all_allowed and len(operation.qubits) == len(self.qubits)
        ):
            raise ValueError(f'Unsupported operation: {operation!r}')

        if operation in self.controlled_gateset:
            if len(operation.qubits) > self._max_parallel_c:
                raise ValueError(
                    'Too many qubits acted on in parallel by a controlled gate operation'
                )
            for p in operation.qubits:
                for q in operation.qubits:
                    if self.distance(p, q) > self._control_radius:
                        raise ValueError(f"Qubits {p!r}, {q!r} are too far away")

    def validate_moment(self, moment: ops.Moment):
        """Raises an error if the given moment is invalid on this device.

        Args:
            moment: The moment to validate

        Raises:
            ValueError: If the given moment is invalid
        """
        super().validate_moment(moment)

        CATEGORIES = {
            'Z': (ops.ZPowGate,),
            'XY': (
                ops.XPowGate,
                ops.YPowGate,
                ops.PhasedXPowGate,
            ),
            'controlled': (
                ops.CNotPowGate,
                ops.CZPowGate,
                ops.CCXPowGate,
                ops.CCZPowGate,
            ),
            'measure': (ops.MeasurementGate,),
        }

        categorized_ops: DefaultDict = collections.defaultdict(list)
        for op in moment.operations:
            assert isinstance(op, (ops.GateOperation, ops.ParallelGateOperation))
            for k, v in CATEGORIES.items():
                assert isinstance(v, tuple)
                gate = _subgate_if_parallel_gate(op.gate)
                if isinstance(gate, v):
                    categorized_ops[k].append(op)

        for k in ['Z', 'XY', 'controlled']:
            if len(set(_subgate_if_parallel_gate(op.gate) for op in categorized_ops[k])) > 1:
                raise ValueError(f"Non-identical simultaneous {k} gates")

        num_parallel_xy = sum([len(op.qubits) for op in categorized_ops['XY']])
        num_parallel_z = sum([len(op.qubits) for op in categorized_ops['Z']])
        has_measurement = len(categorized_ops['measure']) > 0
        controlled_qubits_lists = [op.qubits for op in categorized_ops['controlled']]

        if sum([len(l) for l in controlled_qubits_lists]) > self._max_parallel_c:
            raise ValueError("Too many qubits acted on by controlled gates")
        if controlled_qubits_lists and (num_parallel_xy or num_parallel_z):
            raise ValueError(
                "Can't perform non-controlled operations at same time as controlled operations"
            )
        if self._are_qubit_lists_too_close(*controlled_qubits_lists):
            raise ValueError("Interacting controlled operations")

        if num_parallel_z > self._max_parallel_z:
            raise ValueError("Too many simultaneous Z gates")

        if num_parallel_xy > self._max_parallel_xy and num_parallel_xy != len(self.qubits):
            raise ValueError("Bad number of simultaneous XY gates")

        if has_measurement:
            if controlled_qubits_lists or num_parallel_z or num_parallel_xy:
                raise ValueError("Measurements can't be simultaneous with other operations")

    def _are_qubit_lists_too_close(self, *qubit_lists: Iterable[raw_types.Qid]) -> bool:
        if len(qubit_lists) < 2:
            return False
        if len(qubit_lists) == 2:
            a, b = qubit_lists
            return any(self.distance(p, q) <= self._control_radius for p in a for q in b)
        return any(
            self._are_qubit_lists_too_close(a, b) for a, b in itertools.combinations(qubit_lists, 2)
        )

    def can_add_operation_into_moment(self, operation: ops.Operation, moment: ops.Moment) -> bool:
        """Determines if it's possible to add an operation into a moment.

        An operation can be added if the moment with the operation added is valid.

        Args:
            operation: The operation being added.
            moment: The moment being transformed.

        Returns:
            Whether or not the moment will validate after adding the operation.

        Raises:
            ValueError: If either of the given moment or operation is invalid
        """
        if not super().can_add_operation_into_moment(operation, moment):
            return False
        try:
            self.validate_moment(moment.with_operation(operation))
        except:
            return False
        return True

    def validate_circuit(self, circuit: circuits.AbstractCircuit):
        """Raises an error if the given circuit is invalid on this device.

        A circuit is invalid if any of its moments are invalid or if there is a
        non-empty moment after a moment with a measurement.

        Args:
            circuit: The circuit to validate

        Raises:
            ValueError: If the given circuit can't be run on this device
        """
        super().validate_circuit(circuit)

        # Measurements must be in the last non-empty moment
        has_measurement_occurred = False
        for moment in circuit:
            if has_measurement_occurred:
                if len(moment.operations) > 0:
                    raise ValueError("Non-empty moment after measurement")
            for operation in moment.operations:
                if isinstance(operation.gate, ops.MeasurementGate):
                    has_measurement_occurred = True

    def _value_equality_values_(self) -> Any:
        return (
            self._measurement_duration,
            self._gate_duration,
            self._max_parallel_z,
            self._max_parallel_xy,
            self._max_parallel_c,
            self._control_radius,
            self.qubits,
        )

    def __repr__(self) -> str:
        return (
            'cirq.NeutralAtomDevice('
            f'measurement_duration={self._measurement_duration!r}, '
            f'gate_duration={self._gate_duration!r}, '
            f'max_parallel_z={self._max_parallel_z!r}, '
            f'max_parallel_xy={self._max_parallel_xy!r}, '
            f'max_parallel_c={self._max_parallel_c!r}, '
            f'control_radius={self._control_radius!r}, '
            f'qubits={sorted(self.qubits)!r})'
        )

    def neighbors_of(self, qubit: 'cirq.GridQubit') -> Iterable['cirq.GridQubit']:
        """Returns the qubits that the given qubit can interact with."""
        possibles = [
            GridQubit(qubit.row + 1, qubit.col),
            GridQubit(qubit.row - 1, qubit.col),
            GridQubit(qubit.row, qubit.col + 1),
            GridQubit(qubit.row, qubit.col - 1),
        ]
        return [e for e in possibles if e in self.qubits]

    def distance(self, p: 'cirq.Qid', q: 'cirq.Qid') -> float:
        p = cast(GridQubit, p)
        q = cast(GridQubit, q)
        return sqrt((p.row - q.row) ** 2 + (p.col - q.col) ** 2)

    def __str__(self) -> str:
        diagram = circuits.TextDiagramDrawer()

        for q in self.qubits:
            diagram.write(q.col, q.row, str(q))
            for q2 in self.neighbors_of(q):
                diagram.grid_line(q.col, q.row, q2.col, q2.row)

        return diagram.render(horizontal_spacing=3, vertical_spacing=2, use_unicode_characters=True)


"""Tests related to creating ingest definition"""

import json
import os
import unittest

from rf.models import Scene
from rf.ingest.landsat8_ingest import get_landsat8_layer

class Landsat8LayerTestCase(unittest.TestCase):
    """Test that we can create a layer from Landsat 8 scenes"""

    def setUp(self):
        cwd = os.path.abspath(os.path.dirname(__file__))
        scene_path = os.path.join(cwd, 'data', 'scene.json')
        with open(scene_path) as fh:
            self.scene = Scene.from_dict(json.load(fh))

    def test_create_layer(self):
        """Minimal test to verify that a layer can be created"""
        layer = get_landsat8_layer(self.scene)
        num_sources = len(layer.sources)
        self.assertEqual(
            num_sources, 11, 'Found {} sources, expected 11'.format(num_sources)
        )


# Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import numpy as np
import tensorflow as tf

NUM_CLASSES = 10
EMBEDDING_DIM = 7

def model_fn(features, labels, mode, params):
    # build model
    global_step = tf.train.get_global_step()

    embedding_table = tf.get_variable('embedding_table', shape=(NUM_CLASSES, EMBEDDING_DIM), dtype=tf.float32)

    embeddings = tf.nn.embedding_lookup(embedding_table, features)

    # lstm model
    batch_size = params['train_batch_size']
    sequence_length = params['sequence_length']

    cell = tf.nn.rnn_cell.BasicLSTMCell(EMBEDDING_DIM)
    outputs, final_state = tf.nn.dynamic_rnn(cell, embeddings, dtype=tf.float32)

    # flatten the batch and sequence dimensions
    flattened = tf.reshape(outputs, (-1, EMBEDDING_DIM))
    flattened_logits = tf.layers.dense(flattened, NUM_CLASSES)

    logits = tf.reshape(flattened_logits, (-1, sequence_length, NUM_CLASSES))

    predictions = tf.multinomial(flattened_logits, num_samples=1)
    loss = None
    train_op = None

    if mode == tf.estimator.ModeKeys.TRAIN:
        # define loss
        loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(labels=labels, logits=logits))

        # define train_op
        optimizer = tf.train.RMSPropOptimizer(learning_rate=0.05)

        # wrapper to make the optimizer work with TPUs
        if params['use_tpu']:
            optimizer = tf.contrib.tpu.CrossShardOptimizer(optimizer)

        train_op = optimizer.minimize(loss, global_step=global_step)

    if params['use_tpu']:
        # TPU version of EstimatorSpec
        return tf.contrib.tpu.TPUEstimatorSpec(
            mode=mode,
            predictions=predictions,
            loss=loss,
            train_op=train_op)
    else:
        return tf.estimator.EstimatorSpec(
            mode=mode,
            predictions=predictions,
            loss=loss,
            train_op=train_op)

def train_input_fn(params={}):
    # make some fake data of labels
    data_length = 100
    x = np.random.randint(0, NUM_CLASSES, data_length)
    y = np.random.randint(0, NUM_CLASSES, data_length)

    x_tensor = tf.constant(x, dtype=tf.int32)
    y_tensor = tf.constant(y, dtype=tf.int32)

    dataset = tf.data.Dataset.from_tensors((x_tensor, y_tensor))
    dataset = dataset.repeat()

    # TPUs need to know the full shape of tensors
    # so we use a fixed sequence length
    sequence_length = params.get('sequence_length', 5)

    def get_sequences(x_tensor, y_tensor):
        index = tf.random_uniform([1], minval=0, maxval=data_length-sequence_length, dtype=tf.int32)[0]

        x_sequence = x_tensor[index:index+sequence_length]
        y_sequence = y_tensor[index:index+sequence_length]

        return (x_sequence, y_sequence)

    dataset = dataset.map(get_sequences)

    # TPUEstimator passes params when calling input_fn
    batch_size = params.get('train_batch_size', 16)
    dataset = dataset.batch(batch_size, drop_remainder=True)

    # TPUs need to know all dimensions when the graph is built
    # Datasets know the batch size only when the graph is run
    def set_shapes(features, labels):
        features_shape = features.get_shape().merge_with([batch_size, sequence_length])
        labels_shape = labels.get_shape().merge_with([batch_size, sequence_length])

        features.set_shape(features_shape)
        labels.set_shape(labels_shape)

        return features, labels

    dataset = dataset.map(set_shapes)
    dataset = dataset.prefetch(tf.contrib.data.AUTOTUNE)

    return dataset

def main(args):
    # pass the args as params so the model_fn can use
    # the TPU specific args
    params = vars(args)

    if args.use_tpu:
        # additional configs required for using TPUs
        tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver(args.tpu)
        tpu_config = tf.contrib.tpu.TPUConfig(
            num_shards=8, # using Cloud TPU v2-8
            iterations_per_loop=args.save_checkpoints_steps)

        # use the TPU version of RunConfig
        config = tf.contrib.tpu.RunConfig(
            cluster=tpu_cluster_resolver,
            model_dir=args.model_dir,
            tpu_config=tpu_config,
            save_checkpoints_steps=args.save_checkpoints_steps,
            save_summary_steps=100)

        # TPUEstimator
        estimator = tf.contrib.tpu.TPUEstimator(
            model_fn=model_fn,
            config=config,
            params=params,
            train_batch_size=args.train_batch_size,
            eval_batch_size=32,
            export_to_tpu=False)
    else:
        config = tf.estimator.RunConfig(model_dir=args.model_dir)

        estimator = tf.estimator.Estimator(
            model_fn,
            config=config,
            params=params)

    estimator.train(train_input_fn, max_steps=args.max_steps)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()

    parser.add_argument(
        '--model-dir',
        type=str,
        default='/tmp/tpu-template',
        help='Location to write checkpoints and summaries to.  Must be a GCS URI when using Cloud TPU.')
    parser.add_argument(
        '--max-steps',
        type=int,
        default=1000,
        help='The total number of steps to train the model.')
    parser.add_argument(
        '--sequence-length',
        type=int,
        default=5,
        help='The sequence length for an LSTM model.')
    parser.add_argument(
        '--train-batch-size',
        type=int,
        default=16,
        help='The training batch size.  The training batch is divided evenly across the TPU cores.')
    parser.add_argument(
        '--save-checkpoints-steps',
        type=int,
        default=100,
        help='The number of training steps before saving each checkpoint.')
    parser.add_argument(
        '--use-tpu',
        action='store_true',
        help='Whether to use TPU.')
    parser.add_argument(
        '--tpu',
        default=None,
        help='The name or GRPC URL of the TPU node.  Leave it as `None` when training on AI Platform.')

    args, _ = parser.parse_known_args()

    main(args)


'''
TorMySQL: presents a Tornado and asyncio Future-based API and greenlet for non-blocking access to MySQL.

The MIT License (MIT)

Copyright (c) 2014, 2015 TorMySQL contributors

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
'''

from .client import Client
from .cursor import Cursor, DictCursor, SSCursor, SSDictCursor
from .pool import ConnectionPool
from .cursor import CursorNotReadAllDataError, CursorNotIterError
from .pool import ConnectionPoolClosedError, ConnectionPoolUsedError, ConnectionNotFoundError, ConnectionNotUsedError, ConnectionUsedError, WaitConnectionTimeoutError
from .log import set_log
from . import helpers

version = "0.4.3"
version_info = (0, 4, 3)

def connect(*args, **kwargs):
    client = Client(*args, **kwargs)
    from .platform import current_ioloop
    current_ioloop()
    return client.connect()

Connection = connect


"""

Guidelines for enum classes:
    1. Write members names extensively, with no abbreviation, i.e., 'Watt' instead of 'W'.
    2. Attributes should follow the International System of Units (SI) [https://en.wikipedia.org/wiki/International_System_of_Units], i.e., for power the attribute is 'W'.
    3. Do not use multipliers such as 'Kilowatt'.
        3.1 Exceptions to this rule are: 'Kilometer', 'Kilogram'.
        3.2 In case of an exception, the simple form should be avoided altogether, e.g., given the 'Kilometer' is an Unit, then 'Meter' should not be used.

"""

import enum

@enum.unique
class DisplayNames(str, enum.Enum):
    ElectricityOutput = "ElectricityOutput"
    ElectricityInput = "ElectricityInput"

@enum.unique
class LoadTypes(str, enum.Enum):
    Any = "Any"

    Electricity = "Electricity"
    Irradiance = "Irradiance"
    Speed = "Speed"
    Heating = "Heating"
    Cooling = "Cooling"

    Volume = "Volume"
    Temperature = "Temperature"
    Time = "Time"

    # Substance
    Gas = "Gas"
    Hydrogen = "Hydrogen"
    Oxygen = "Oxygen"
    Water = "Water"
    WarmWater = "WarmWater"

    Price = "Price"

@enum.unique
class Units(str, enum.Enum):
    # Unphysical
    Any = "-"
    Percent = "%"

    # Power
    Watt = "W"
    kW = "kW"
    kWh_per_timestep = "kWh per timestep"

    # Power per area
    Wm2 = "W per square meter"
    Whm2 = "Wh per square meter"

    # Speed
    MeterPerSecond = "m/s"

    # Energy
    Wh = "Wh"
    kWh = "kWh"

    # Volume
    Liter = "L"

    # Volume per time
    l_per_timestep = "Liter per timestep"

    # Mass
    kg = "kg"

    # Mass flow
    kg_per_sec = "kg/s"

    # Degrees
    Celsius = "°C"
    Kelvin = 'K'

    # Degrees
    Degrees = "Degrees"

    # Time
    Seconds = "s"

    # Cost
    c_per_kWh = "Cents per kWh"


import argparse
import collections
import json
import os

import numpy as np
import torch
import yaml

__all__ = [
    "load_config",
    "save_config",
    "flatten_dict",
    "sanitize_dict",
    "update_namespace",
    "extract",
    "s2b",
    "g",
]

# Load config file
def load_yaml(f_path):
    with open(f_path, "r") as stream:
        return yaml.safe_load(stream)

def load_json(f_path):
    with open(f_path, "r") as f:
        return json.load(f)

def load_config(path, flatten=True):
    _, ext = os.path.splitext(path)

    assert ext in [
        ".json",
        ".yaml",
        ".yml",
    ], f"Only support yaml and json config, but '{ext}' given."
    if ext == "json":
        cfg = load_json(path)
    else:
        cfg = load_yaml(path)

    if cfg is None:
        cfg = dict()

    if flatten:
        cfg = flatten_dict(cfg)
    return cfg

# Dump config file
def save_json(obj, f_path):
    with open(f_path, "w") as f:
        json.dump(obj, f, ensure_ascii=False, indent=4)

def save_yaml(obj, f_path):
    with open(f_path, "w") as f:
        yaml.dump(obj, f)

def save_config(obj, path, ext=None):
    _, fext = os.path.splitext(path)
    if fext.startswith("."):
        fext = fext[1:]
    if fext != "":
        assert (
            ext == None or fext == ext
        ), f"Extension conflict between '{path}' and '{ext}'."
        ext = fext

    if ext in ["yaml", "yml"]:
        save_yaml(obj, path)
    else:
        save_json(obj, path)

# Utils
def flatten_dict(d, keep_parent=False, sep="_", parent_key=""):
    """Flatten dict to only one nest

    Args:
        d (dict): dictionary to flatten
        keep_parent (bool, optional): If True, keep parent's key name, and keys should all be str. Defaults to False.
        sep (str, optional): Effective only keep_parent=True, separator between keys. Defaults to "_".
        parent_key (str, optional): For recursive call. Defaults to "".

    Returns:
        dict: flattened dict
    """
    items = []
    for k, v in d.items():
        new_key = parent_key + sep + k if parent_key and keep_parent else k
        if isinstance(v, collections.abc.MutableMapping):
            items.extend(
                flatten_dict(v, keep_parent, parent_key=new_key, sep=sep).items()
            )
        else:
            items.append((new_key, v))

    items_key = [i[0] for i in items]
    assert len(items_key) == len(set(items_key))

    return dict(items)

def sanitize_dict(params, to_str=True, none_fill="N/A"):
    """Convert all items into tensorboard supported values or str

    Args:
        params (dict): dict to sanitize
        to_str (bool, optional): If True, turn all items to string. Defaults to True.

    Returns:
        dict: sanitized dict
    """
    items = []
    for k in params.keys():
        # numpy to float
        if isinstance(params[k], (np.bool_, np.integer, np.floating)):
            items.append([k, params[k].item()])
        elif isinstance(params[k], np.ndarray):
            items.append([k, str(params[k].tolist())])
        # torch to float
        elif isinstance(params[k], torch.Tensor):
            items.append([k, str(params[k].tolist())])
        # None to str
        elif params[k] is None:
            items.append([k, none_fill])
        # Others to str
        elif type(params[k]) not in [bool, int, float, str, torch.Tensor]:
            items.append([k, str(params[k])])
        else:
            items.append([k, params[k]])

        # All to str
        if to_str:
            items[-1][-1] = str(items[-1][-1])

    return dict(items)

def update_namespace(args, dictionary, overwrite=True, rest=False):
    """update Namespace with given dictionary

    Args:
        args (Namespace): Namespace to be updated
        dictionary (dict): dictionary
        overwrite (bool, optional): If True, All Namespace value will overwritten by dictionary value. Otherwise, only Namespace with None will be overwritten. Defaults to True.
        rest: Effective only if overwrite=True. If True, add keys in dictionary but not in args into args. Otherwise raise an error.

    Returns:
        Namespace
    """
    dict_args = vars(args)

    if overwrite:
        dict_args.update(dictionary)
    else:
        for k, v in dict_args.items():
            if v is not None:
                pass
            elif k in dictionary:
                dict_args[k] = dictionary[k]
        for k, v in dictionary.items():
            if k not in dict_args:
                if rest:
                    dict_args[k] = v
                else:
                    raise KeyError(f"no key {k}")

    args = argparse.Namespace(**dict_args)
    return args

def extract(s, delimit="-", num=0):
    """Extract the num_th word from string s

    Args:
        s (str): string to be parsed
        delimit (str, optional): delimiter. Defaults to "-".
        num (int, optional): . Defaults to 0.

    Returns:
        (str, List[str])
    """
    s_list = s.split(delimit)
    first = s_list[num]
    s_list.pop(num)
    s_rest = delimit.join(s_list)
    return first, s_rest

# argparse type
def s2b(v):
    if isinstance(v, bool):
        return v
    if v.lower() in ("yes", "true", "t", "y", "1"):
        return True
    elif v.lower() in ("no", "false", "f", "n", "0"):
        return False
    else:
        raise argparse.ArgumentTypeError("Boolean value expected.")

# template generator for params.py
def g(template, name_list, placeholder="{}"):
    items = []
    for name in name_list:
        t = []
        t.append(template[0].replace(placeholder, name))
        t.append(template[1].replace(placeholder, name))
        t.extend(template[2:])
        items.append(t)
    return items


# -*- coding:utf-8 -*-

import unittest

class TestZip(unittest.TestCase):
    TESTDATA = [
            ("aabbb" , "a2b3"),
            ("aaaa", "a4"),
            ("abc", "abc"),
            ("abcdd","abcdd")
            ]
    def setUp(self):
        self.judge = Zipper()

    def testsame(self):
        for src, exp in self.TESTDATA:
            self.assertEqual(self.judge.zipString(src),exp)

class Zipper:
    def zipString(self, iniString):
        # write code here
        record = []
        prevchar = None
        prevlen = 0

        for letter in iniString:
            if letter == prevchar:
                prevlen += 1
            else:
                if prevlen > 0:
                    record.append({prevchar : prevlen})
                prevlen = 1
                prevchar = letter
        if prevlen > 0:
            record.append({prevchar : prevlen})
        newstring = ''
        for item in record:
            for key,value in item.iteritems():
                newstring += "{}{}".format(key,value)
        return newstring if len(newstring) < len(iniString) else iniString

if __name__ == '__main__':
    unittest.main()


#   Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function

import unittest
import numpy as np
import sys
sys.path.append("..")
from op_test import OpTest, skip_check_grad_ci
import paddle
import paddle.fluid.core as core
import paddle.fluid as fluid
from paddle.fluid import compiler, Program, program_guard
from paddle.fluid.framework import convert_np_dtype_to_dtype_

paddle.enable_static()

class TestAny8DOp(OpTest):
    def setUp(self):
        self.set_npu()
        self.op_type = "reduce_any"
        self.place = paddle.NPUPlace(0)
        self.inputs = {
            'X': np.random.randint(0, 2,
                                   (2, 5, 3, 2, 2, 3, 4, 2)).astype("bool")
        }
        self.attrs = {'dim': (3, 5, 4)}
        self.outputs = {'Out': self.inputs['X'].any(axis=self.attrs['dim'])}

    def set_npu(self):
        self.__class__.use_npu = True

    def test_check_output(self):
        self.check_output_with_place(self.place)

class TestAnyOpWithDim(OpTest):
    def setUp(self):
        self.set_npu()
        self.op_type = "reduce_any"
        self.place = paddle.NPUPlace(0)
        self.inputs = {'X': np.random.randint(0, 2, (5, 6, 10)).astype("bool")}
        self.attrs = {'dim': [1]}
        self.outputs = {'Out': self.inputs['X'].any(axis=1)}

    def set_npu(self):
        self.__class__.use_npu = True

    def test_check_output(self):
        self.check_output_with_place(self.place)

class TestAny8DOpWithDim(OpTest):
    def setUp(self):
        self.set_npu()
        self.op_type = "reduce_any"
        self.place = paddle.NPUPlace(0)
        self.inputs = {
            'X': np.random.randint(0, 2,
                                   (2, 5, 3, 2, 2, 3, 4, 2)).astype("bool")
        }
        self.attrs = {'dim': (3, 6)}
        self.outputs = {'Out': self.inputs['X'].any(axis=self.attrs['dim'])}

    def set_npu(self):
        self.__class__.use_npu = True

    def test_check_output(self):
        self.check_output_with_place(self.place)

class TestAnyOpWithKeepDim(OpTest):
    def setUp(self):
        self.set_npu()
        self.op_type = "reduce_any"
        self.place = paddle.NPUPlace(0)
        self.inputs = {'X': np.random.randint(0, 2, (5, 6, 10)).astype("bool")}
        self.attrs = {'dim': (1), 'keep_dim': True}
        self.outputs = {
            'Out': np.expand_dims(
                self.inputs['X'].any(axis=self.attrs['dim']), axis=1)
        }

    def set_npu(self):
        self.__class__.use_npu = True

    def test_check_output(self):
        self.check_output_with_place(self.place)

class TestAny8DOpWithKeepDim(OpTest):
    def setUp(self):
        self.set_npu()
        self.op_type = "reduce_any"
        self.place = paddle.NPUPlace(0)
        self.inputs = {
            'X': np.random.randint(0, 2,
                                   (2, 5, 3, 2, 2, 3, 4, 2)).astype("bool")
        }
        self.attrs = {'dim': (1), 'keep_dim': True}
        self.outputs = {
            'Out': np.expand_dims(
                self.inputs['X'].any(axis=self.attrs['dim']), axis=1)
        }

    def set_npu(self):
        self.__class__.use_npu = True

    def test_check_output(self):
        self.check_output_with_place(self.place)

if __name__ == '__main__':
    unittest.main()


# -*- coding: utf-8 -*-
# Copyright (c) 2018, Awab Abdoun and Mohammed Elamged and Contributors
# See license.txt
from __future__ import unicode_literals

import frappe
import unittest

class TestWorkstation(unittest.TestCase):
	pass


# Copyright (c) 2009-2019 The Regents of the University of Michigan
# This file is part of the HOOMD-blue project, released under the BSD 3-Clause License.

# Maintainer: jglaser / All Developers are free to add commands for new features

R""" Potentials between special pairs of particles

Special pairs are used to implement interactions between designated pairs of particles.
They act much like bonds, except that the interaction potential is typically a pair potential,
such as LJ.

By themselves, special pairs that have been specified in an initial configuration do nothing. Only when you
specify an force (i.e. special_pairs.lj), are forces actually calculated between the
listed particles.
"""

from hoomd import _hoomd
from hoomd.md import _md
from hoomd.md import force;
from hoomd.md import bond;
import hoomd;

import math;
import sys;

class coeff:
    R""" Define special_pair coefficients.

    The coefficients for all special pair potentials are specified using this class. Coefficients are
    specified per pair type.

    There are two ways to set the coefficients for a particular special_pair potential.
    The first way is to save the special_pair potential in a variable and call :py:meth:`set()` directly.
    See below for an example of this.

    The second method is to build the coeff class first and then assign it to the
    special_pair potential. There are some advantages to this method in that you could specify a
    complicated set of special_pair potential coefficients in a separate python file and import
    it into your job script.

    Example::

        my_coeffs = hoomd.md.special_pair.coeff();
        special_pair_force.pair_coeff.set('pairtype1', epsilon=1, sigma=1)
        special_pair_force.pair_coeff.set('backbone', epsilon=1.2, sigma=1)

    """

    ## \internal
    # \brief Initializes the class
    # \details
    # The main task to be performed during initialization is just to init some variables
    # \param self Python required class instance variable
    def __init__(self):
        self.values = {};
        self.default_coeff = {}

    ## \var values
    # \internal
    # \brief Contains the vector of set values in a dictionary

    ## \var default_coeff
    # \internal
    # \brief default_coeff['coeff'] lists the default value for \a coeff, if it is set

    ## \internal
    # \brief Sets a default value for a given coefficient
    # \details
    # \param name Name of the coefficient to for which to set the default
    # \param value Default value to set
    #
    # Some coefficients have reasonable default values and the user should not be burdened with typing them in
    # all the time. set_default_coeff() sets
    def set_default_coeff(self, name, value):
        self.default_coeff[name] = value;

    def set(self, type, **coeffs):
        R""" Sets parameters for special_pair types.

        Args:
            type (str): Type of special_pair (or a list of type names)
            coeffs: Named coefficients (see below for examples)

        Calling :py:meth:`set()` results in one or more parameters being set for a special_pair type. Types are identified
        by name, and parameters are also added by name. Which parameters you need to specify depends on the special_pair
        potential you are setting these coefficients for, see the corresponding documentation.

        All possible special_pair types as defined in the simulation box must be specified before executing run().
        You will receive an error if you fail to do so. It is not an error, however, to specify coefficients for
        special_pair types that do not exist in the simulation. This can be useful in defining a potential field for many
        different types of special_pairs even when some simulations only include a subset.

        Examples::

            my_special_pair_force.special_pair_coeff.set('pair1', epsilon=1, sigma=1)
            my_special_pair_force.pair_coeff.set('pair2', epsilon=0.5, sigma=0.7)
            my_special_pair_force.pair_coeff.set(['special_pairA','special_pairB'], epsilon=0, sigma=1)

        Note:
            Single parameters can be updated. If both ``k`` and ``r0`` have already been set for a particle type,
            then executing ``coeff.set('polymer', r0=1.0)`` will update the value of ``r0`` and leave the other
            parameters as they were previously set.

        """
        hoomd.util.print_status_line();

        # listify the input
        type = hoomd.util.listify(type)

        for typei in type:
            self.set_single(typei, coeffs);

    ## \internal
    # \brief Sets a single parameter
    def set_single(self, type, coeffs):
        type = str(type);

        # create the type identifier if it hasn't been created yet
        if (not type in self.values):
            self.values[type] = {};

        # update each of the values provided
        if len(coeffs) == 0:
            hoomd.context.msg.error("No coefficients specified\n");
        for name, val in coeffs.items():
            self.values[type][name] = val;

        # set the default values
        for name, val in self.default_coeff.items():
            # don't override a coeff if it is already set
            if not name in self.values[type]:
                self.values[type][name] = val;

    ## \internal
    # \brief Verifies that all values are set
    # \details
    # \param self Python required self variable
    # \param required_coeffs list of required variables
    #
    # This can only be run after the system has been initialized
    def verify(self, required_coeffs):
        # first, check that the system has been initialized
        if not hoomd.init.is_initialized():
            hoomd.context.msg.error("Cannot verify special_pair coefficients before initialization\n");
            raise RuntimeError('Error verifying force coefficients');

        # get a list of types from the particle data
        ntypes = hoomd.context.current.system_definition.getPairData().getNTypes();
        type_list = [];
        for i in range(0,ntypes):
            type_list.append(hoomd.context.current.system_definition.getPairData().getNameByType(i));

        valid = True;
        # loop over all possible types and verify that all required variables are set
        for i in range(0,ntypes):
            type = type_list[i];

            if type not in self.values.keys():
                hoomd.context.msg.error("Pair type " +str(type) + " not found in pair coeff\n");
                valid = False;
                continue;

            # verify that all required values are set by counting the matches
            count = 0;
            for coeff_name in self.values[type].keys():
                if not coeff_name in required_coeffs:
                    hoomd.context.msg.notice(2, "Notice: Possible typo? Force coeff " + str(coeff_name) + " is specified for type " + str(type) + \
                          ", but is not used by the special pair force\n");
                else:
                    count += 1;

            if count != len(required_coeffs):
                hoomd.context.msg.error("Special pair type " + str(type) + " is missing required coefficients\n");
                valid = False;

        return valid;

    ## \internal
    # \brief Gets the value of a single %special_pair %force coefficient
    # \detail
    # \param type Name of special_pair type
    # \param coeff_name Coefficient to get
    def get(self, type, coeff_name):
        if type not in self.values.keys():
            hoomd.context.msg.error("Bug detected in force.coeff. Please report\n");
            raise RuntimeError("Error setting special_pair coeff");

        return self.values[type][coeff_name];

    ## \internal
    # \brief Return metadata
    def get_metadata(self):
        return self.values

## \internal
# \brief Base class for special pair potentials
#
# A special pair in hoomd.* reflects a PotentialSpecialPair in c++. It is responsible
# for all high-level management that happens behind the scenes for hoomd
# writers. 1) The instance of the c++ bond force itself is tracked and added to the
# System 2) methods are provided for disabling the force from being added to the
# net force on each particle
class _special_pair(force._force):
    ## \internal
    # \brief Constructs the bond potential
    #
    # \param name name of the bond potential instance
    #
    # Initializes the cpp_force to None.
    # If specified, assigns a name to the instance
    # Assigns a name to the force in force_name;
    def __init__(self, name=None):
        # initialize the base class
        force._force.__init__(self, name);

        self.cpp_force = None;

        # setup the coefficient vector (use bond coefficients for that)
        self.pair_coeff = coeff();

        self.enabled = True;

    def update_coeffs(self):
        coeff_list = self.required_coeffs;
        # check that the force coefficients are valid
        if not self.pair_coeff.verify(coeff_list):
           hoomd.context.msg.error("Not all force coefficients are set\n");
           raise RuntimeError("Error updating force coefficients");

        # set all the params
        ntypes = hoomd.context.current.system_definition.getPairData().getNTypes();
        type_list = [];
        for i in range(0,ntypes):
            type_list.append(hoomd.context.current.system_definition.getPairData().getNameByType(i));

        for i in range(0,ntypes):
            # build a dict of the coeffs to pass to proces_coeff
            coeff_dict = {};
            for name in coeff_list:
                coeff_dict[name] = self.pair_coeff.get(type_list[i], name);

            param = self.process_coeff(coeff_dict);
            self.cpp_force.setParams(i, param);

    ## \internal
    # \brief Get metadata
    def get_metadata(self):
        data = force._force.get_metadata(self)

        # make sure coefficients are up-to-date
        self.update_coeffs()

        data['pair_coeff'] = self.pair_coeff
        return data

class lj(_special_pair):
    R""" LJ special pair potential.

    Args:
        name (str): Name of the special_pair instance.

    :py:class:`lj` specifies a Lennard-Jones potential energy between the two particles in each defined pair.

    This is useful for implementing e.g. special 1-4 interactions in all-atom force fields.

    The pair potential uses the standard LJ definition.

    .. math::
        :nowrap:

        \begin{eqnarray*}
        V_{\mathrm{LJ}}(r)  = & 4 \varepsilon \left[ \left( \frac{\sigma}{r} \right)^{12} -
                          \alpha \left( \frac{\sigma}{r} \right)^{6} \right] & r < r_{\mathrm{cut}} \\
                            = & 0 & r \ge r_{\mathrm{cut}} \\
        \end{eqnarray*}

    where :math:`\vec{r}` is the vector pointing from one particle to the other in the bond.

    Coefficients:

    - :math:`\varepsilon` - *epsilon* (in energy units)
    - :math:`\sigma` - *sigma* (in distance units)
    - :math:`\alpha` - *alpha* (unitless) - *optional*: defaults to 1.0
    - :math:`r_{\mathrm{cut}}` - *r_cut* (in distance units)

    Example::

        lj = special_pair.lj(name="my_pair")
        lj.pair_coeff.set('pairtype_1', epsilon=5.4, sigma=0.47, r_cut=1.1)

    Note:
        The energy of special pair interactions is reported in a log quantity **special_pair_lj_energy**, which
        is separate from those of other non-bonded interactions. Therefore, the total energy of nonbonded interactions
        is obtained by adding that of standard and special interactions.

    .. versionadded:: 2.1

    """
    def __init__(self,name=None):
        hoomd.util.print_status_line();

        # initialize the base class
        _special_pair.__init__(self);

        # check that some bonds are defined
        if hoomd.context.current.system_definition.getPairData().getNGlobal() == 0:
            hoomd.context.msg.error("No pairs are defined.\n");
            raise RuntimeError("Error creating special pair forces");

        # create the c++ mirror class
        if not hoomd.context.exec_conf.isCUDAEnabled():
            self.cpp_force = _md.PotentialSpecialPairLJ(hoomd.context.current.system_definition,self.name);
        else:
            self.cpp_force = _md.PotentialSpecialPairLJGPU(hoomd.context.current.system_definition,self.name);

        hoomd.context.current.system.addCompute(self.cpp_force, self.force_name);

        # setup the coefficient options
        self.required_coeffs = ['epsilon','sigma','alpha','r_cut'];
        self.pair_coeff.set_default_coeff('alpha', 1.0);

    def process_coeff(self, coeff):
        r_cut = coeff['r_cut'];
        epsilon = coeff['epsilon'];
        sigma = coeff['sigma'];
        alpha = coeff['alpha'];

        lj1 = 4.0 * epsilon * math.pow(sigma, 12.0);
        lj2 = alpha * 4.0 * epsilon * math.pow(sigma, 6.0);
        r_cut_squared = r_cut * r_cut
        return _hoomd.make_scalar3(lj1, lj2, r_cut_squared);

class coulomb(_special_pair):
    R""" Coulomb special pair potential.

    Args:
        name (str): Name of the special_pair instance.

    :py:class:`coulomb` specifies a Coulomb potential energy between the two particles in each defined pair.

    This is useful for implementing e.g. special 1-4 interactions in all-atom force fields. It uses a standard Coulomb interaction with a scaling parameter. This allows for using this for scaled 1-4 interactions like in OPLS where both the 1-4 LJ and Coulomb interactions are scaled by 0.5.

    .. math::
        :nowrap:

        \begin{eqnarray*}
        V_{\mathrm{Coulomb}}(r)  = & \alpha \cdot \left[ \frac{q_{a}q_{b}}{r} \right] & r < r_{\mathrm{cut}} \\
                            = & 0 & r \ge r_{\mathrm{cut}} \\
        \end{eqnarray*}

    where :math:`\vec{r}` is the vector pointing from one particle to the other in the bond.

    Coefficients:

    - :math:`\alpha` - Coulomb scaling factor (defaults to 1.0)
    - :math:`q_{a}` - charge of particle a (in hoomd charge units)
    - :math:`q_{b}` - charge of particle b (in hoomd charge units)
    - :math:`r_{\mathrm{cut}}` - *r_cut* (in distance units)

    Example::

        coul = special_pair.coulomb(name="myOPLS_style")
        coul.pair_coeff.set('pairtype_1', alpha=0.5, r_cut=1.1)

    Note:
        The energy of special pair interactions is reported in a log quantity **special_pair_coul_energy**, which
        is separate from those of other non-bonded interactions. Therefore, the total energy of non-bonded interactions
        is obtained by adding that of standard and special interactions.

    .. versionadded:: 2.2
    .. versionchanged:: 2.2

    """
    def __init__(self, name=None):
        hoomd.util.print_status_line();

        # initialize the base class
        _special_pair.__init__(self);

        # check that some bonds are defined
        if hoomd.context.current.system_definition.getPairData().getNGlobal() == 0:
            hoomd.context.msg.error("No pairs are defined.\n");
            raise RuntimeError("Error creating special pair forces");

        # create the c++ mirror class
        if not hoomd.context.exec_conf.isCUDAEnabled():
            self.cpp_force = _md.PotentialSpecialPairCoulomb(hoomd.context.current.system_definition,self.name);
        else:
            self.cpp_force = _md.PotentialSpecialPairCoulombGPU(hoomd.context.current.system_definition,self.name);

        hoomd.context.current.system.addCompute(self.cpp_force, self.force_name);

        # setup the coefficient options
        self.required_coeffs = ['alpha', 'r_cut'];
        self.pair_coeff.set_default_coeff('alpha', 1.0);

    def process_coeff(self, coeff):
        r_cut = coeff['r_cut'];
        alpha = coeff['alpha'];

        r_cut_squared = r_cut * r_cut;
        return _hoomd.make_scalar2(alpha, r_cut_squared);


#!/usr/bin/python3
"""
    fasttextRun.py: run fasttext via python interface
    usage: fasttextRun.py -f file [-n N]
    note: default number of N is 10 (10-fold cross validation)
    20180105 erikt(at)xs4all.nl
"""

import fasttext
import os
import random
import splitFile
import sys

COMMAND = sys.argv.pop(0)
DIM = 300
LARGENUMBER = 100000
MINCOUNT = 5
random.seed()
TMPFILENAME = "fasttextRun."+str(os.getpid())+"."+str(random.randint(0,LARGENUMBER))

def makeTrainFile(inFileName,i,n):
    outFileName = TMPFILENAME+".train"
    outFile = open(outFileName,"w")
    for j in range(0,n):
        if j != i:
            inFile = open(inFileName+"."+str(j),"r")
            for line in inFile: outFile.write(line)
            inFile.close()
    outFile.close()
    return(outFileName)

def fasttextRun(inFileName,i,n):
    trainFileName = makeTrainFile(inFileName,i,n)
    modelFileName = TMPFILENAME+".model"
    testFileName = inFileName+"."+str(i)
    classifier = fasttext.supervised(trainFileName,modelFileName,dim=DIM,min_count=MINCOUNT)
    # ,pretrained_vectors="/home/erikt/software/fastText/wiki.nl.vec")
    result = classifier.test(testFileName)
    os.unlink(trainFileName)
    os.unlink(modelFileName+".bin")
    return(result.precision)

def main(argv):
    inFileName, n = splitFile.processOpts(list(argv))
    data = splitFile.readData(inFileName)
    splitFile.writeData(inFileName,data,n)
    accuracyTotal = 0.0
    for i in range(0,n):
        accuracy = fasttextRun(inFileName,i,n)
        accuracyTotal += accuracy
        print("Fold: {0:0d}; Accuracy: {1:0.3f}".format(i,accuracy))
    print("Average accuracy {0:0.3f}".format(accuracyTotal/float(n)))
    return(0)

if __name__ == "__main__":
    sys.exit(main(sys.argv))


# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Remove useless index

Revision ID: 7750037b351a
Revises: f449e5bff5a5
Create Date: 2016-12-17 21:10:27.781900
"""

from alembic import op

revision = "7750037b351a"
down_revision = "f449e5bff5a5"

def upgrade():
    op.drop_index("release_files_name_idx", table_name="release_files")

def downgrade():
    op.create_index("release_files_name_idx", "release_files", ["name"], unique=False)


# coding: utf8

from .tsv_utils import complementary_list, find_label, baseline_df, chi2
from clinicaaddl.tools.deep_learning.iotools import return_logger
from scipy.stats import ttest_ind
import shutil
import pandas as pd
from os import path
import numpy as np
import os
import logging

sex_dict = {'M': 0, 'F': 1}

def create_split(diagnosis, diagnosis_df, n_test,
                 pval_threshold_ttest=0.80, t_val_chi2_threshold=0.0642,
                 ignore_demographics=False, logger=None):
    """
    Split data at the subject-level in training and test set with equivalent age and sex distributions

    :param diagnosis: (str) diagnosis on which the split is done
    :param diagnosis_df: DataFrame with columns including ['participant_id', 'session_id', 'diagnosis']
    :param n_test: (float)
        If >= 1 number of subjects to put in the test set.
        If < 1 proportion of subjects to put in the test set.
    :param pval_threshold_ttest: (float) threshold for the t-test on age
    :param t_val_chi2_threshold:  (float) threshold for the chi2 test on sex
    :param ignore_demographics: (bool): If True the diagnoses are split without taking into account the demographics
    distributions (age, sex).
    :param logger: Logger object from logging library
    :return:
        train_df (DataFrame) subjects in the train set
        test_df (DataFrame) subjects in the test set
    """

    if logger is None:
        logger = logging
        logger.basicConfig(level=logging.DEBUG)

    diagnosis_baseline_df = baseline_df(diagnosis_df)

    if n_test >= 1:
        n_test = int(n_test)
    else:
        n_test = int(n_test * len(diagnosis_baseline_df))

    if not ignore_demographics:
        try:
            sex_label = find_label(diagnosis_baseline_df.columns.values, "sex")
            age_label = find_label(diagnosis_baseline_df.columns.values, "age")
        except ValueError:
            raise ValueError("This dataset do not have age or sex values. "
                             "Please add the flag --ignore_demographics to split "
                             "without trying to balance age or sex distributions.")

        sex = list(diagnosis_baseline_df[sex_label].values)
        age = list(diagnosis_baseline_df[age_label].values)

        idx = np.arange(len(diagnosis_baseline_df))

        flag_selection = True
        n_try = 0

        while flag_selection:
            idx_test = np.random.choice(idx, size=n_test, replace=False)
            idx_test.sort()
            idx_train = complementary_list(idx, idx_test)

            # Find similarity of distribution for the age variable
            if len(set(age)) != 1:
                age_test = [float(age[idx]) for idx in idx_test]
                age_train = [float(age[idx]) for idx in idx_train]

                t_age, p_age = ttest_ind(age_test, age_train)
            else:
                p_age = 1

            # Find the a similar distribution for the sex variable
            if len(set(sex)) != 1:
                sex_test = [sex_dict[sex[idx]] for idx in idx_test]
                sex_train = [sex_dict[sex[idx]] for idx in idx_train]
                T_sex = chi2(sex_test, sex_train)
            else:
                T_sex = 0

            logger.debug("p=%.2f, T=%.4f" % (p_age, T_sex))
            if T_sex < t_val_chi2_threshold and p_age > pval_threshold_ttest:
                flag_selection = False
                test_df = diagnosis_baseline_df.loc[idx_test]
                train_df = diagnosis_baseline_df.loc[idx_train]

            n_try += 1

        logger.info("Split for diagnosis %s was found after %i trials" % (diagnosis, n_try))
    else:
        idx = np.arange(len(diagnosis_baseline_df))
        idx_test = np.random.choice(idx, size=n_test, replace=False)
        idx_test.sort()
        idx_train = complementary_list(idx, idx_test)
        test_df = diagnosis_baseline_df.loc[idx_test]
        train_df = diagnosis_baseline_df.loc[idx_train]

    return train_df, test_df

def split_diagnoses(formatted_data_path,
                    n_test=100, subset_name="test", MCI_sub_categories=True,
                    t_val_threshold=0.0642, p_val_threshold=0.80,
                    ignore_demographics=False, verbose=0):
    """
    Performs a single split for each label independently on the subject level.
    The train folder will contain two lists per diagnosis (baseline and longitudinal),
    whereas the test folder will only include the list of baseline sessions.

    The age and sex distributions between the two sets must be non-significant (according to T-test and chi-square).

    Args:
        formatted_data_path (str): Path to the folder containing data extracted by clinicaaddl tsvtool getlabels.
        n_test (float):
            If > 1, number of subjects to put in set with name 'subset_name'.
            If < 1, proportion of subjects to put in set with name 'subset_name'.
            If 0, no training set is created and the whole dataset is considered as one set with name 'subset_name'.
        subset_name (str): Name of the subset that is complementary to train.
        MCI_sub_categories (bool): If True, manages MCI sub-categories to avoid data leakage.
        t_val_threshold (float): The threshold used for the chi2 test on sex distributions.
        p_val_threshold (float): The threshold used for the T-test on age distributions.
        ignore_demographics (bool): If True the diagnoses are split without taking into account the demographics
            distributions (age, sex).
        verbose (int): level of verbosity.

    Returns:
        writes three files per <label>.tsv file present in formatted_data_path:
            - formatted_data_path/train/<label>.tsv
            - formatted_data_path/train/<label>_baseline.tsv
            - formatted_data_path/<subset_name>/<label>_baseline.tsv
    """
    logger = return_logger(verbose, "split")

    # Read files
    results_path = formatted_data_path

    train_path = path.join(results_path, 'train')
    if path.exists(train_path):
        shutil.rmtree(train_path)
    if n_test > 0:
        os.makedirs(train_path)

    test_path = path.join(results_path, subset_name)
    if path.exists(test_path):
        shutil.rmtree(test_path)
    os.makedirs(test_path)

    diagnosis_df_paths = os.listdir(results_path)
    diagnosis_df_paths = [x for x in diagnosis_df_paths if x.endswith('.tsv')]
    diagnosis_df_paths = [x for x in diagnosis_df_paths if not x.endswith('_baseline.tsv')]

    MCI_special_treatment = False

    if 'MCI.tsv' in diagnosis_df_paths and n_test > 0:
        if MCI_sub_categories:
            diagnosis_df_paths.remove('MCI.tsv')
            MCI_special_treatment = True
        elif 'sMCI.tsv' in diagnosis_df_paths or 'pMCI.tsv' in diagnosis_df_paths:
            logger.warning("MCI special treatment was deactivated though MCI subgroups were found."
                           "Be aware that it may cause data leakage in transfer learning tasks.")

    # The baseline session must be kept before or we are taking all the sessions to mix them
    for diagnosis_df_path in diagnosis_df_paths:
        diagnosis_df = pd.read_csv(path.join(results_path, diagnosis_df_path),
                                   sep='\t')
        interest_columns = diagnosis_df.columns.values
        diagnosis = diagnosis_df_path.split('.')[0]
        logger.info("Running split for diagnosis %s" % diagnosis)
        if n_test > 0:
            train_df, test_df = create_split(diagnosis, diagnosis_df, n_test=n_test,
                                             t_val_chi2_threshold=t_val_threshold,
                                             pval_threshold_ttest=p_val_threshold,
                                             ignore_demographics=ignore_demographics,
                                             logger=logger)
            # Save baseline splits
            train_df = train_df[interest_columns]
            train_df.to_csv(path.join(train_path, str(diagnosis) + '_baseline.tsv'), sep='\t', index=False)
            test_df = test_df[interest_columns]
            test_df.to_csv(path.join(test_path, str(diagnosis) + '_baseline.tsv'), sep='\t', index=False)

            # Retrieve all sessions for the training set
            complete_train_df = pd.DataFrame()
            for idx in train_df.index.values:
                subject = train_df.loc[idx, 'participant_id']
                subject_df = diagnosis_df[diagnosis_df.participant_id == subject]
                complete_train_df = pd.concat([complete_train_df, subject_df])

            complete_train_df.to_csv(path.join(train_path, str(diagnosis) + '.tsv'), sep='\t', index=False)

        else:
            diagnosis_baseline_df = baseline_df(diagnosis_df)
            test_df = diagnosis_baseline_df[interest_columns]
            test_df.to_csv(path.join(test_path, str(diagnosis) + '_baseline.tsv'), sep='\t', index=False)

    if MCI_special_treatment:

        # Extraction of MCI subjects without intersection with the sMCI / pMCI train
        diagnosis_df = pd.read_csv(path.join(results_path, 'MCI.tsv'), sep='\t')
        MCI_df = diagnosis_df.set_index(['participant_id', 'session_id'])
        baseline_MCI_df = baseline_df(MCI_df, set_index=False)
        supplementary_diagnoses = []

        if n_test > 1:
            n_test = int(n_test)
        else:
            n_test = int(n_test * len(baseline_MCI_df))

        logger.debug('Before subjects removal for MCI special treatment')
        if n_test > 1:
            n_test = int(n_test)
        else:
            n_test = int(n_test * len(baseline_MCI_df))

        sub_df = diagnosis_df.reset_index().groupby('participant_id')['session_id'].nunique()
        logger.debug('%i subjects, %i scans' % (len(sub_df), len(diagnosis_df)))

        if 'sMCI.tsv' in diagnosis_df_paths:
            sMCI_baseline_train_df = pd.read_csv(path.join(train_path, 'sMCI_baseline.tsv'), sep='\t')
            sMCI_baseline_test_df = pd.read_csv(path.join(test_path, 'sMCI_baseline.tsv'), sep='\t')
            sMCI_baseline_df = pd.concat([sMCI_baseline_train_df, sMCI_baseline_test_df])
            sMCI_baseline_df.reset_index(drop=True, inplace=True)
            for idx in sMCI_baseline_df.index.values:
                subject = sMCI_baseline_df.loc[idx, 'participant_id']
                MCI_df.drop(subject, inplace=True)
            supplementary_diagnoses.append('sMCI')

            logger.debug('Removed %i subjects based on sMCI label' % len(sMCI_baseline_df))
            sub_df = MCI_df.reset_index().groupby('participant_id')['session_id'].nunique()
            logger.debug('%i subjects, %i scans' % (len(sub_df), len(MCI_df)))

        if 'pMCI.tsv' in diagnosis_df_paths:
            pMCI_baseline_train_df = pd.read_csv(path.join(train_path, 'pMCI_baseline.tsv'), sep='\t')
            pMCI_baseline_test_df = pd.read_csv(path.join(test_path, 'pMCI_baseline.tsv'), sep='\t')
            pMCI_baseline_df = pd.concat([pMCI_baseline_train_df, pMCI_baseline_test_df])
            pMCI_baseline_df.reset_index(drop=True, inplace=True)
            for idx in pMCI_baseline_df.index.values:
                subject = pMCI_baseline_df.loc[idx, 'participant_id']
                MCI_df.drop(subject, inplace=True)
            supplementary_diagnoses.append('pMCI')

            logger.debug('Removed %i subjects based on pMCI label' % len(pMCI_baseline_df))
            sub_df = MCI_df.reset_index().groupby('participant_id')['session_id'].nunique()
            logger.debug('%i subjects, %i scans' % (len(sub_df), len(MCI_df)))

        if len(supplementary_diagnoses) == 0:
            raise ValueError('The MCI_sub_categories flag is not needed as there are no intersections with'
                             'MCI subcategories.')

        # Construction of supplementary train
        supplementary_train_df = pd.DataFrame()
        for diagnosis in supplementary_diagnoses:
            sup_baseline_train_df = pd.read_csv(path.join(train_path, diagnosis + '_baseline.tsv'), sep='\t')
            supplementary_train_df = pd.concat([supplementary_train_df, sup_baseline_train_df])
            sub_df = supplementary_train_df.reset_index().groupby('participant_id')['session_id'].nunique()
            logger.debug('supplementary_train_df %i subjects, %i scans' % (len(sub_df), len(supplementary_train_df)))

        supplementary_train_df.reset_index(drop=True, inplace=True)

        # MCI selection
        MCI_df.reset_index(inplace=True)
        diagnosis_baseline_df = baseline_df(MCI_df)
        if not ignore_demographics:
            sex_label = find_label(diagnosis_baseline_df.columns.values, "sex")
            age_label = find_label(diagnosis_baseline_df.columns.values, "age")

            sex = list(diagnosis_baseline_df[sex_label].values)
            age = list(diagnosis_baseline_df[age_label].values)

            sup_train_sex = list(supplementary_train_df[sex_label].values)
            sup_train_age = list(supplementary_train_df[age_label].values)

            sup_train_sex = [sex_dict[x] for x in sup_train_sex]
            sup_train_age = [float(x) for x in sup_train_age]

            idx = np.arange(len(diagnosis_baseline_df))

            flag_selection = True
            n_try = 0

            while flag_selection:
                idx_test = np.random.choice(idx, size=n_test, replace=False)
                idx_test.sort()
                idx_train = complementary_list(idx, idx_test)

                # Find similarity of distribution for the age variable
                if len(set(age)) != 1:
                    age_test = [float(age[idx]) for idx in idx_test]
                    age_train = [float(age[idx]) for idx in idx_train]

                    t_age, p_age = ttest_ind(age_test, age_train)
                else:
                    p_age = 1

                # Find similarity of distribution for the sex variable
                if len(set(sex)) != 1:
                    sex_test = [sex_dict[sex[idx]] for idx in idx_test]
                    sex_train = [sex_dict[sex[idx]] for idx in idx_train]
                    T_sex = chi2(sex_test, sex_train)
                else:
                    T_sex = 0

                logger.debug("p=%.2f, T=%.4f" % (p_age, T_sex))
                if T_sex < t_val_threshold and p_age > p_val_threshold:
                    flag_selection = False
                    MCI_baseline_test_df = diagnosis_baseline_df.loc[idx_test]
                    train_df = diagnosis_baseline_df.loc[idx_train]
                    MCI_baseline_train_df = pd.concat([train_df, supplementary_train_df])
                    logger.debug('Supplementary train df %i' % len(supplementary_train_df))
                    MCI_baseline_train_df.reset_index(drop=True, inplace=True)

                n_try += 1

            logger.info('Split for diagnosis MCI was found after %i trials' % n_try)
        else:
            idx = np.arange(len(diagnosis_baseline_df))
            idx_test = np.random.choice(idx, size=n_test, replace=False)
            idx_test.sort()
            idx_train = complementary_list(idx, idx_test)

            MCI_baseline_test_df = diagnosis_baseline_df.loc[idx_test]
            train_df = diagnosis_baseline_df.loc[idx_train]
            MCI_baseline_train_df = pd.concat([train_df, supplementary_train_df])
            MCI_baseline_train_df.reset_index(drop=True, inplace=True)

        # Write selection of MCI
        MCI_baseline_train_df = MCI_baseline_train_df[interest_columns]
        MCI_baseline_train_df.to_csv(path.join(train_path, 'MCI_baseline.tsv'), sep='\t', index=False)
        MCI_baseline_test_df = MCI_baseline_test_df[interest_columns]
        MCI_baseline_test_df.to_csv(path.join(test_path, 'MCI_baseline.tsv'), sep='\t', index=False)

        # Retrieve all sessions for the training set
        MCI_complete_train_df = pd.DataFrame()
        for idx in MCI_baseline_train_df.index.values:
            subject = MCI_baseline_train_df.loc[idx, 'participant_id']
            subject_df = diagnosis_df[diagnosis_df.participant_id == subject]
            MCI_complete_train_df = pd.concat([MCI_complete_train_df, subject_df])

        MCI_complete_train_df.to_csv(path.join(train_path, 'MCI.tsv'), sep='\t', index=False)


# -*- encoding: utf-8 -*-
# pylint: disable=E0203,E1101,C0111
"""
@file
@brief Runtime operator.
"""
import numpy
from ._op import OpRun

class ConstantOfShape(OpRun):

    atts = {'value': numpy.array([0], dtype=numpy.float32)}

    def __init__(self, onnx_node, desc=None, **options):
        OpRun.__init__(self, onnx_node, desc=desc,
                       expected_attributes=ConstantOfShape.atts,
                       **options)
        self.cst = (self.value[0]
                    if isinstance(self.value, numpy.ndarray)
                    else self.value)
        if not isinstance(self.cst, (float, numpy.float32, numpy.float64)):
            raise TypeError("cst must be a real not {}".format(type(self.cst)))

    def _run(self, data):  # pylint: disable=W0221
        res = numpy.full(tuple(data), self.cst)
        return (res, )


#!/usr/bin/env python3
import argparse
import json
import os
from patrace import (
    InputFile,
    OutputFile,
    Call,
    CreateInt32Value,
)

class Arg:
    def __init__(self, type, name, value):
        self.type = type
        self.name = name
        self.value = value

    def get(self):
        arg = self.type(self.value)
        if self.name:
            arg.mName = self.name
        return arg

class Function:
    def __init__(self, name, args):
        self.name = name
        self.args = args

    def write(self, output, tid):
        call = Call(self.name)
        call.thread_id = tid

        for arg in self.args[1:]:
            call.args.push_back(arg.get())

        call.return_value = self.args[0].get()

        output.WriteCall(call)

class Remapper:
    def __init__(self):
        self.num_calls_remapped = 0

    def run(self, input, output):
        # Modify header, if we are remaping the default tid
        header = json.loads(input.jsonHeader)
        default_tid = header['defaultTid']
        output.jsonHeader = json.dumps(header)

        print('Searching for relevant calls...')
        call_lists = {
            'eglMakeCurrent': [],
            'eglCreateContext': [],
            'eglDestroyContext': [],
        }

        context_calls = []
        highest_thread_id = -1
        for call in input.Calls():
            highest_thread_id = max(call.thread_id, highest_thread_id)
            # call_list = call_lists.get(call.name, None)
            if call.name in list(call_lists.keys()):
                context_calls.append({
                    'name': call.name,
                    'tid': call.thread_id,
                    'params': call.GetArgumentsDict().copy(),
                    'retval': call.GetReturnValue(),
                    'number': call.number,
                })
            # if call_list is not None:
            #     call_list.append({
            #         'call_name': call.name,
            #         'tid': call.thread_id,
            #         'params': call.GetArgumentsDict(),
            #         'retval': call.GetReturnValue(),
            #         'number': call.number,
            #     })
        num_threads = highest_thread_id + 1

        print("Renumbering context ids...")
        # Sometimes, contexts can get the same pointer values
        # Hence, the contexts pointers will not be unique. Therefor,
        # we create an unique, sequential id.
        context_sequential_id = 1
        # Maps original context id with sequential context id.
        contexts_idmap = {0: 0}
        for call in context_calls:
            if call['name'] == 'eglCreateContext':
                contexts_idmap[call['retval']] = context_sequential_id
                call['retval'] = context_sequential_id
                context_sequential_id += 1
            elif call['name'] == 'eglDestroyContext':
                old_id = call['params']['ctx']
                seq_id = contexts_idmap[old_id]
                del contexts_idmap[old_id]
                call['params']['ctx'] = seq_id
            elif call['name'] == 'eglMakeCurrent':
                # Change ctx parameter to our new sequential id
                call['params']['ctx'] = contexts_idmap[call['params']['ctx']]

        print("Finding relevant context and surfaces...")
        make_current_args = [
            (call['params']['draw'], call['params']['ctx'])
            for call in context_calls
            if (
                call['name'] == 'eglMakeCurrent'
                # Excluding the following test made things work for GunJack
                # call['tid'] in [default_tid, 0]
            )
        ]

        import pprint
        pprint.pprint(make_current_args)

        surfaces = []
        contexts = []

        for draw, ctx in make_current_args:
            if draw:
                surfaces.append(draw)

            if ctx:
                contexts.append(ctx)

        # Find all relevant shared contexts
        shared_contexts = []

        for context in contexts:
            for context_call in context_calls:
                if context_call['name'] != 'eglCreateContext':
                    continue

                if context_call['retval'] == context:
                    shared_contexts.append(context_call['params']['share_context'])

        for share_context in shared_contexts:
            contexts.append(share_context)

        contexts = set(contexts)
        surfaces = set(surfaces)
        print("Surfaces {}".format(surfaces))
        print("Contexts: {}".format(contexts))

        class Thread:
            def __init__(self):
                self.current_ctx_seq = 0
                self.current_ctx_old = 0
                self.remap = 0

        threads = [Thread() for i in range(num_threads)]
        # Used to indicate if inside a relevant "eglMakeCurrent-block"

        print("Remap calls...")

        contextid_to_use = None

        contexts_idmap = {0: 0}
        context_sequential_id = 1
        active_thread = -1
        for call in input.Calls():
            current_thread = call.thread_id
            thread_switch = False
            if active_thread != current_thread:
                thread_switch = True
                active_thread = current_thread

            if call.name == 'eglCreateContext':
                oldid = call.GetReturnValue()
                contexts_idmap[oldid] = context_sequential_id
                if context_sequential_id in contexts:
                    contextid_to_use = oldid
                    print("We will map all calls of the context:", contextid_to_use)
                    self.remap(call, default_tid)
                context_sequential_id += 1
            elif call.name == 'eglDestroyContext':
                ad = call.GetArgumentsDict()
                oldid = ad['ctx']
                # seqid = contexts_idmap[oldid]
                del contexts_idmap[oldid]
            elif (
                call.name.startswith('eglCreateWindowSurface') or
                call.name == 'eglCreatePbufferSurface'
            ):
                if call.GetReturnValue() in surfaces:
                    self.remap(call, default_tid)
            elif call.name == 'eglDestroySurface':
                ad = call.GetArgumentsDict()
                if ad['surface'] in surfaces:
                    self.remap(call, default_tid)
            elif call.name == 'eglMakeCurrent':
                t = threads[call.thread_id]
                ad = call.GetArgumentsDict()
                t.current_dpy = ad['dpy']
                t.current_draw = ad['draw']
                t.current_read = ad['read']
                t.current_ctx_old = ad['ctx']
                t.current_ctx_seq = contexts_idmap[ad['ctx']]

                if t.current_ctx_seq in contexts:
                    # call.SetArgument(3, contextid_to_use)
                    t.remap = True

                if ad['ctx'] == 0:
                    t.remap = False

            if threads[call.thread_id].remap:
                # If a context is already active on the default thread
                # We need to inject an eglMakeCurrent the first time
                if thread_switch and call.name != 'eglMakeCurrent':
                    t = threads[call.thread_id]
                    Function(
                        'eglMakeCurrent', [
                            Arg(CreateInt32Value, '', 1),
                            Arg(CreateInt32Value, 'dpy', t.current_dpy),
                            Arg(CreateInt32Value, 'draw', t.current_draw),
                            Arg(CreateInt32Value, 'read', t.current_read),
                            Arg(CreateInt32Value, 'ctx', t.current_ctx_old),
                        ]
                    ).write(output, default_tid)

                self.remap(call, default_tid)

            output.WriteCall(call)

    def remap(self, call, newtid):
        call.thread_id = newtid
        self.num_calls_remapped += 1

def remap(oldfile, newfile):
    remapper = Remapper()

    if not os.path.exists(oldfile):
        print("File does not exists: {}".format(oldfile))
        return

    with InputFile(oldfile) as input:
        with OutputFile(newfile) as output:
            remapper.run(input, output)

    return remapper.num_calls_remapped

def main():
    parser = argparse.ArgumentParser(description='Automatically remap thread ids in a .pat trace. This should be used if an eglContext is used by more threads than the default thread.')
    parser.add_argument('oldfile', help='Path to the .pat trace file')
    parser.add_argument('newfile', help='New .pat file to create')

    args = parser.parse_args()
    num = remap(args.oldfile, args.newfile)

    print("Number of calls remapped {num}".format(num=num))

if __name__ == '__main__':
    main()