#   Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# TODO: define statistical functions of a tensor

import numpy as np
from ..fluid.framework import Variable
from ..fluid.layer_helper import LayerHelper
from ..fluid.framework import core, in_dygraph_mode
from ..fluid import layers
from .search import where
from ..fluid.data_feeder import convert_dtype, check_variable_and_dtype, check_type, check_dtype
import paddle
from paddle import _C_ops

__all__ = []

def mean(x, axis=None, keepdim=False, name=None):
    """
    Computes the mean of the input tensor's elements along ``axis``.

    Args:
        x (Tensor): The input Tensor with data type float32, float64.
        axis (int|list|tuple, optional): The axis along which to perform mean
            calculations. ``axis`` should be int, list(int) or tuple(int). If
            ``axis`` is a list/tuple of dimension(s), mean is calculated along
            all element(s) of ``axis`` . ``axis`` or element(s) of ``axis``
            should be in range [-D, D), where D is the dimensions of ``x`` . If
            ``axis`` or element(s) of ``axis`` is less than 0, it works the
            same way as :math:`axis + D` . If ``axis`` is None, mean is
            calculated over all elements of ``x``. Default is None.
        keepdim (bool, optional): Whether to reserve the reduced dimension(s)
            in the output Tensor. If ``keepdim`` is True, the dimensions of
            the output Tensor is the same as ``x`` except in the reduced
            dimensions(it is of size 1 in this case). Otherwise, the shape of
            the output Tensor is squeezed in ``axis`` . Default is False.
        name (str, optional): Name for the operation (optional, default is None).
            For more information, please refer to :ref:`api_guide_Name`.

    Returns:
        Tensor, results of average along ``axis`` of ``x``, with the same data
        type as ``x``.

    Examples:
        .. code-block:: python

            import paddle

            x = paddle.to_tensor([[[1., 2., 3., 4.],
                                   [5., 6., 7., 8.],
                                   [9., 10., 11., 12.]],
                                  [[13., 14., 15., 16.],
                                   [17., 18., 19., 20.],
                                   [21., 22., 23., 24.]]])
            out1 = paddle.mean(x)
            # [12.5]
            out2 = paddle.mean(x, axis=-1)
            # [[ 2.5  6.5 10.5]
            #  [14.5 18.5 22.5]]
            out3 = paddle.mean(x, axis=-1, keepdim=True)
            # [[[ 2.5]
            #   [ 6.5]
            #   [10.5]]
            #  [[14.5]
            #   [18.5]
            #   [22.5]]]
            out4 = paddle.mean(x, axis=[0, 2])
            # [ 8.5 12.5 16.5]
    """

    if isinstance(axis, int):
        axis = [axis]
    reduce_all = True if axis is None \
        or len(axis)==0 \
        or len(axis) == len(x.shape) else False
    if axis is None or len(axis) == 0:
        axis = [0]

    if in_dygraph_mode():
        return _C_ops.reduce_mean(x, 'dim', axis, 'keep_dim', keepdim,
                                  'reduce_all', reduce_all)

    check_variable_and_dtype(x, 'x/input', ['float32', 'float64'],
                             'mean/reduce_mean')
    check_type(axis, 'axis/dim', (int, list, tuple), 'mean/reduce_mean')
    if isinstance(axis, (list, tuple)):
        for item in axis:
            check_type(item, 'elements of axis/dim', (int), 'mean/reduce_mean')

    helper = LayerHelper('mean', **locals())
    attrs = {'dim': axis, 'keep_dim': keepdim, 'reduce_all': reduce_all}
    out = helper.create_variable_for_type_inference(x.dtype)
    helper.append_op(
        type='reduce_mean', inputs={'X': x}, outputs={'Out': out}, attrs=attrs)
    return out

def var(x, axis=None, unbiased=True, keepdim=False, name=None):
    """
    Computes the variance of ``x`` along ``axis`` .

    Args:
        x (Tensor): The input Tensor with data type float32, float64.
        axis (int|list|tuple, optional): The axis along which to perform
            variance calculations. ``axis`` should be int, list(int) or
            tuple(int). If ``axis`` is a list/tuple of dimension(s), variance
            is calculated along all element(s) of ``axis`` . ``axis`` or
            element(s) of ``axis`` should be in range [-D, D), where D is the
            dimensions of ``x`` . If ``axis`` or element(s) of ``axis`` is less
            than 0, it works the same way as :math:`axis + D` . If ``axis`` is
            None, variance is calculated over all elements of ``x``. Default
            is None.
        unbiased (bool, optional): Whether to use the unbiased estimation. If
            ``unbiased`` is True, the divisor used in the computation is
            :math:`N - 1`, where :math:`N` represents the number of elements
            along ``axis`` , otherwise the divisor is :math:`N`. Default is True.
        keepdim (bool, optional): Whether to reserve the reduced dimension(s)
            in the output Tensor. If ``keepdim`` is True, the dimensions of
            the output Tensor is the same as ``x`` except in the reduced
            dimensions(it is of size 1 in this case). Otherwise, the shape of
            the output Tensor is squeezed in ``axis`` . Default is False.
        name (str, optional): Name for the operation (optional, default is None).
            For more information, please refer to :ref:`api_guide_Name`.

    Returns:
        Tensor, results of variance along ``axis`` of ``x``, with the same data
        type as ``x``.

    Examples:
        .. code-block:: python

            import paddle

            x = paddle.to_tensor([[1.0, 2.0, 3.0], [1.0, 4.0, 5.0]])
            out1 = paddle.var(x)
            # [2.66666667]
            out2 = paddle.var(x, axis=1)
            # [1.         4.33333333]
    """
    if not in_dygraph_mode():
        check_variable_and_dtype(x, 'x', ['float32', 'float64'], 'var')

    u = mean(x, axis, True, name)
    out = paddle.sum((x - u)**2, axis, keepdim=keepdim, name=name)

    n = paddle.cast(paddle.numel(x), x.dtype) \
        / paddle.cast(paddle.numel(out), x.dtype)
    if unbiased:
        one_const = paddle.ones([1], x.dtype)
        n = where(n > one_const, n - 1., one_const)
    out /= n
    return out

def std(x, axis=None, unbiased=True, keepdim=False, name=None):
    """
    Computes the standard-deviation of ``x`` along ``axis`` .

    Args:
        x (Tensor): The input Tensor with data type float32, float64.
        axis (int|list|tuple, optional): The axis along which to perform
            standard-deviation calculations. ``axis`` should be int, list(int)
            or tuple(int). If ``axis`` is a list/tuple of dimension(s),
            standard-deviation is calculated along all element(s) of ``axis`` .
            ``axis`` or element(s) of ``axis`` should be in range [-D, D),
            where D is the dimensions of ``x`` . If ``axis`` or element(s) of
            ``axis`` is less than 0, it works the same way as :math:`axis + D` .
            If ``axis`` is None, standard-deviation is calculated over all
            elements of ``x``. Default is None.
        unbiased (bool, optional): Whether to use the unbiased estimation. If
            ``unbiased`` is True, the standard-deviation is calculated via the
            unbiased estimator. If ``unbiased`` is True,  the divisor used in
            the computation is :math:`N - 1`, where :math:`N` represents the
            number of elements along ``axis`` , otherwise the divisor is
            :math:`N`. Default is True.
        keepdim (bool, optional): Whether to reserve the reduced dimension(s)
            in the output Tensor. If ``keepdim`` is True, the dimensions of
            the output Tensor is the same as ``x`` except in the reduced
            dimensions(it is of size 1 in this case). Otherwise, the shape of
            the output Tensor is squeezed in ``axis`` . Default is False.
        name (str, optional): Name for the operation (optional, default is None).
            For more information, please refer to :ref:`api_guide_Name`.

    Returns:
        Tensor, results of standard-deviation along ``axis`` of ``x``, with the
        same data type as ``x``.

    Examples:
        .. code-block:: python

            import paddle

            x = paddle.to_tensor([[1.0, 2.0, 3.0], [1.0, 4.0, 5.0]])
            out1 = paddle.std(x)
            # [1.63299316]
            out2 = paddle.std(x, axis=1)
            # [1.       2.081666]
    """
    if not in_dygraph_mode():
        check_variable_and_dtype(x, 'x', ['float32', 'float64'], 'std')

    out = var(**locals())
    return paddle.sqrt(out)

def numel(x, name=None):
    """
    Returns the number of elements for a tensor, which is a int64 Tensor with shape [1] in static mode
    or a scalar value in imperative mode

    Args:
        x (Tensor): The input Tensor, it's data type can be bool, float16, float32, float64, int32, int64.

    Returns:
        Tensor: The number of elements for the input Tensor.

    Examples:
        .. code-block:: python

            import paddle

            x = paddle.full(shape=[4, 5, 7], fill_value=0, dtype='int32')
            numel = paddle.numel(x) # 140

    """
    if in_dygraph_mode():
        return _C_ops.size(x)

    if not isinstance(x, Variable):
        raise TypeError("x must be a Tensor in numel")
    helper = LayerHelper('numel', **locals())
    out = helper.create_variable_for_type_inference(
        dtype=core.VarDesc.VarType.INT64)
    helper.append_op(type='size', inputs={'Input': x}, outputs={'Out': out})
    return out

def median(x, axis=None, keepdim=False, name=None):
    """
    Compute the median along the specified axis.

    Args:
        x (Tensor): The input Tensor, it's data type can be bool, float16, float32, float64, int32, int64.
        axis (int, optional): The axis along which to perform median calculations ``axis`` should be int.
            ``axis`` should be in range [-D, D), where D is the dimensions of ``x`` .
            If ``axis`` is less than 0, it works the same way as :math:`axis + D`.
            If ``axis`` is None, median is calculated over all elements of ``x``. Default is None.
        keepdim (bool, optional): Whether to reserve the reduced dimension(s)
            in the output Tensor. If ``keepdim`` is True, the dimensions of
            the output Tensor is the same as ``x`` except in the reduced
            dimensions(it is of size 1 in this case). Otherwise, the shape of
            the output Tensor is squeezed in ``axis`` . Default is False.
        name (str, optional): Name for the operation (optional, default is None).
            For more information, please refer to :ref:`api_guide_Name`.

    Returns:
        Tensor, results of median along ``axis`` of ``x``. If data type of ``x`` is float64, data type of results will be float64, otherwise data type will be float32.

    Examples:
        .. code-block:: python

            import paddle

            x = paddle.arange(12).reshape([3, 4])
            # x is [[0 , 1 , 2 , 3 ],
            #       [4 , 5 , 6 , 7 ],
            #       [8 , 9 , 10, 11]]

            y1 = paddle.median(x)
            # y1 is [5.5]

            y2 = paddle.median(x, axis=0)
            # y2 is [4., 5., 6., 7.]

            y3 = paddle.median(x, axis=1)
            # y3 is [1.5, 5.5, 9.5]

            y4 = paddle.median(x, axis=0, keepdim=True)
            # y4 is [[4., 5., 6., 7.]]

    """
    if not isinstance(x, Variable):
        raise TypeError("In median, the input x should be a Tensor.")
    is_flatten = axis is None
    dims = len(x.shape)
    if is_flatten:
        x = paddle.flatten(x)
        axis = 0
    else:
        if not isinstance(axis, int) or not (axis < dims and axis >= -dims):
            raise ValueError(
                "In median, axis should be none or an integer in range [-rank(x), rank(x))."
            )
        if axis < 0:
            axis += dims
    sz = x.shape[axis]
    kth = sz >> 1
    tensor_topk, idx = paddle.topk(x, kth + 1, axis=axis, largest=False)
    dtype = 'float64' if x.dtype == core.VarDesc.VarType.FP64 else 'float32'
    if sz & 1 == 0:
        out_tensor = paddle.slice(
            tensor_topk, axes=[axis], starts=[kth - 1],
            ends=[kth]) + paddle.slice(
                tensor_topk, axes=[axis], starts=[kth], ends=[kth + 1])
        out_tensor = paddle.cast(out_tensor, dtype=dtype) / 2
    else:
        out_tensor = paddle.cast(
            paddle.slice(
                tensor_topk, axes=[axis], starts=[kth], ends=[kth + 1]),
            dtype=dtype)
    if not keepdim or is_flatten:
        if not is_flatten:
            newshape = x.shape[:axis] + x.shape[axis + 1:]
        elif not keepdim:
            newshape = [1]
        else:
            newshape = [1] * dims
    else:
        newshape = out_tensor.shape
    out_tensor = out_tensor.reshape(newshape, name=name)
    return out_tensor


# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# http://doc.scrapy.org/en/latest/topics/items.html

from scrapy.item import Item, Field

class XiaobaiheItem(Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    username = Field()
    text = Field()
    url = Field()


import pyaf.Bench.TS_datasets as tsds
import pyaf.tests.artificial.process_artificial_dataset as art

art.process_dataset(N = 32 , FREQ = 'D', seed = 0, trendtype = "MovingMedian", cycle_length = 12, transform = "None", sigma = 0.0, exog_count = 20, ar_order = 0);


""" Tests for the various cli programs """

from pyontutils.integration_test_helper import _TestCliBase, Folders

class TestCli(Folders, _TestCliBase):
    commands = (
        ['googapis', '--help'],
        ['graphml-to-ttl', '--help'],
        ['necromancy', '--help'],
        ['ontload', '--help'],
        ['overlaps', '--help'],
        ['qnamefix', '--help'],
        ['scigraph-codegen', '--help'],
        ['scig', '--help'],
        ['ttlfmt', '--help'],
    )


#!/usr/bin/python

import math
import matplotlib.pyplot as plt
from graphtheory.structures.edges import Edge
from graphtheory.structures.graphs import Graph
from graphtheory.structures.factory import GraphFactory
from graphtheory.structures.points import Point
from graphtheory.forests.treeplot import TreePlot
from graphtheory.forests.treeplot import TreePlotRadiusAngle

V = 20
gf = GraphFactory(Graph)
G = gf.make_tree(V)
#G.show()
assert G.e() == V-1

algorithm = TreePlotRadiusAngle(G)
algorithm.run()
#print ( algorithm.point_dict )   # (radius, angle)

D = dict()   # node ---> point on the plane
for node in algorithm.point_dict:
    (radius, angle) = algorithm.point_dict[node]
    D[node] = Point(radius * math.cos(angle), radius * math.sin(angle))
#print ( D )

for edge in G.iteredges():
    x = [D[edge.source].x, D[edge.target].x]
    y = [D[edge.source].y, D[edge.target].y]
    plt.plot(x, y, 'k-')   # black line

x = [D[node].x for node in G.iternodes()]
y = [D[node].y for node in G.iternodes()]
plt.plot(x, y, 'bo')   # blue circle

plt.title("Random tree")
plt.xlabel("x")
plt.ylabel("y")
plt.show()

# EOF


import pandas as pd
import numpy as np
import joblib
import Levenshtein
import argparse
import ast
from scipy import stats

from src import nlp_preprocessing

def preprocess_txt(txt: str):
    """Executa preprocessamento textual padrão"""
    cleaned_txt = nlp_preprocessing.clean_text(txt)
    token_txt = nlp_preprocessing.custom_tokenizer(cleaned_txt)
    return " ".join(token_txt)

def get_top_docs(query, cleaned_doc_list, doc_titles, get_titles=True):
    """Gera as recomedações a partir de uma query e listas de referência"""
    cleaned_query = preprocess_txt(query)
    dists = [Levenshtein.distance(cleaned_query, doc)
             for doc in cleaned_doc_list]

    mask = np.array(dists).argsort()[:10]
    if get_titles:
        return doc_titles.iloc[mask].tolist()
    else:
        return doc_titles.iloc[mask].index.tolist()

def load_data():
    """Carrega os dados"""
    df = pd.concat([
        pd.read_pickle("data/train_query.pickle"),
        pd.read_pickle("data/test_query.pickle")
    ])
    return df

def series_mode(serie: pd.Series):
    """Calcula a moda de uma série"""
    return stats.mode(serie)[0][0]

def remove_duplicates(df, group="product_id",
                      num_cols=["price", "weight", "minimum_quantity"],
                      cat_cols=["title", "concatenated_tags"]) -> pd.DataFrame:
    """Função que remove os registros duplicados juntando os por média e moda
    a depender dos tipos de coluna"""
    mode_stats = {col: series_mode for col in cat_cols}
    mean_stats = {col: "mean" for col in num_cols}
    agg_stats = dict(**mode_stats, **mean_stats)
    return df.groupby(group).agg(agg_stats)

def make_predictions(query, clf_model):
    """Função que realiza as recomendações com predição de categoria
    majoritária"""
    df = load_data()
    prod_titles = (df[["product_id", "title"]].
                   drop_duplicates().set_index("product_id")["title"])
    cleaned_prod_titles = [preprocess_txt(txt) for txt in prod_titles]
    prod_id_select = get_top_docs(query,
                                  cleaned_prod_titles,
                                  prod_titles,
                                  False)
    selected_df = df.loc[df["product_id"].isin(prod_id_select)]
    selected_df = remove_duplicates(selected_df)
    predicted_cats = clf_model.predict(selected_df)
    major_cat = stats.mode(predicted_cats)[0][0]
    print(major_cat)
    for _id, title in selected_df["title"].iteritems():
        print(f"{_id} - {title}")

# função auxiliar do modelo de classificação
def select_txt(X: pd.DataFrame, col: str):
    return X[col]

# função auxiliar do modelo de classificação
def select_base_features(X: pd.DataFrame):
    return X[["price", "weight", "minimum_quantity"]]

def load_args() -> pd.DataFrame:
    """Função de carregamento de configurações.

    Returns:
        pd.DataFrame: resultado a ser categorizado.
    """
    # criando os atributos que vão ser recebidos e parseando-os
    parser = argparse.ArgumentParser()
    parser.add_argument("-c",
                        "--category",
                        help="Texto de registro a ser categorizado",
                        type=str)
    parser.add_argument("-r",
                        "--recommendation",
                        help="Sistema de recomendação de produtos",
                        type=str)
    args = parser.parse_args()

    # extraindo dos atributos recebidos o registro a ser categorizado
    # e adequando-o para a predição
    if args.category is not None:
        product_dict = ast.literal_eval(args.category)
        product_df = pd.Series(product_dict).to_frame().T
    else:
        product_df = None

    return product_df, args.recommendation

def predict_single_category(df, clf_model):
    product_category = clf_model.predict(df)[0]
    print(product_category)

def main():
    # carregando o modelo
    rf_clf_pipeline = joblib.load("assets/category_rf_clf_pipeline.joblib")
    # carregando o registro a ser categorizado
    product_df, query = load_args()
    # fazendo a previsão da categoria
    if product_df is not None:
        predict_single_category(product_df, rf_clf_pipeline)
    if query is not None:
        make_predictions(query, rf_clf_pipeline)

if __name__ == "__main__":
    main()


import re
import os
import struct
import argparse
import collections
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
from nltk.stem.wordnet import WordNetLemmatizer
from tensorflow.core.example import example_pb2
from utils import is_num

parser = argparse.ArgumentParser()

# Path
parser.add_argument('--dataset', type=str, choices=['restaurant', 'beer'], default='restaurant')
parser.add_argument('--vocab_fname', type=str, default='./data/vocab.txt')

args = parser.parse_args()

"""
Preprocessing script file
"""

def tokenize_sent(sent, lemtzr, stopword):
    tokens = tokenize(sent.strip().lower())
    tokens = re.sub(r'[^A-Za-z0-9]+',' ', ' '.join(tokens))
    tokens = [tok for tok in tokens.split() if tok not in stopword]
    tokens = [tok if not is_num(tok) else '<NUM>' for tok in tokens]
    tokens = [lemtzr.lemmatize(tok) for tok in tokens]
    return tokens

def tokenize_train_file(fname):
    """
    Tokenize the raw train data(unlabeled).
    """
    split_fname = fname.split('/')
    new_fname = '/'.join([el if idx != len(split_fname) - 1 else 'parsed_' + el for idx, el in enumerate(split_fname)])
    if os.path.exists(new_fname): return new_fname

    with open(fname, 'r', encoding='utf8') as f:
        ls = f.readlines()

    parsed_data = []
    lemtzr = WordNetLemmatizer()
    stopword = stopwords.words('english')

    for line in ls:
        tokens = tokenize_sent(line, lemtzr, stopword)
        parsed_data.append(tokens)

    save_file(parsed_data, new_fname)
    return new_fname

def tokenize_labeled_test_file(fname, label_fname):
    """
    Tokenize the raw test data (labelled).
    """
    split_fname = fname.split('/')
    new_fname = '/'.join([el if idx != len(split_fname) - 1 else 'parsed_' + el for idx, el in enumerate(split_fname)])
    label_map_fname = '/'.join([el if idx != len(split_fname) - 1 else 'label_map.txt' for idx, el in enumerate(split_fname)])
    if os.path.exists(new_fname): return label_map_fname, new_fname

    with open(fname, 'r', encoding='utf8') as f1, open(label_fname, 'r', encoding='utf8') as f2:
        ls1, ls2 = f1.readlines(), f2.readlines()

    parsed_data = []
    lemtzr = WordNetLemmatizer()
    stopword = stopwords.words('english')

    for line in ls1:
        tokens = tokenize_sent(line, lemtzr, stopword)
        parsed_data.append(tokens)

    assert len(ls1) == len(ls2) == len(parsed_data)

    new_parsed, new_ls2 = [], []
    for parsed, label in zip(parsed_data, ls2):
        if 'Positive' in label or 'Neutral' in label:
            continue
        new_parsed.append(parsed)
        new_ls2.append(label)

    assert len(new_parsed) == len(new_ls2)
    parsed_data, ls2 = new_parsed, new_ls2

    label_text = list(set([tok for line in ls2 for tok in line.strip().split()]))

    label_map = dict()
    print("Label for this dataset with assigned index is as follows.")
    for idx, label in enumerate(label_text):
        print('{}: {}'.format(label, idx))
        label_map[label] = idx
    with open(label_map_fname, 'w') as f:
        for key,val in label_map.items():
            f.write("{}  {} ||| ".format(key, val))

    for idx, data in enumerate(parsed_data):
        labels = ls2[idx].strip().split()
        assert all([label in list(label_map.keys()) for label in labels])
        parsed_data[idx].insert(0, '|||')
        for label in labels:
            parsed_data[idx].insert(0, str(label_map[label]))

    save_file(parsed_data, new_fname)
    return label_map_fname, new_fname

def build_vocab(parsed_train_fname, vocab_file, vocab_size=30000):
    """
    Build vocab based on frequency of each word in train set.
    Save vocab file and return vocab list.
    """
    if os.path.exists(vocab_file):
        with open(vocab_file, 'r', encoding='utf8') as f:
            ls = f.readlines()
            assert len(ls) == vocab_size
        vocab = [line.strip() for line in ls]
        return vocab

    with open(parsed_train_fname, 'r', encoding='utf8') as f:
        ls = f.readlines()
        tokens = [tok for line in ls for tok in line.strip().split()]
    counts = dict(collections.Counter(tokens))

    import operator
    vocab = sorted(counts.items(), key=operator.itemgetter(1), reverse=True)
    print("TOTAL VOCAB SIZE: {}".format(len(vocab)))

    for idx,tok in enumerate(vocab):
        if tok[1] <= 10:
            print("WORDS MORE THAN 10: {}".format(idx))
            break
    vocab = [tok[0] for tok in vocab][:idx]

    vocab.append('<UNK>')
    vocab.append('<PAD>')

    assert all([isinstance(tok, str) for tok in vocab])
    with open(vocab_file, 'w') as f:
        f.write('\n'.join([tok for tok in vocab]))
    return vocab

def save_file(data, new_fname):
    """
    Change the "raw_fname" into parsed fname, then save "data" into parsed fname.
    """
    assert isinstance(data, list)

    with open(new_fname, 'w') as f: f.write('\n'.join([" ".join(one_sample) for one_sample in data]))

def tokenize(sent):
    assert isinstance(sent, str)
    return word_tokenize(sent)

def make_binary_dataset(fname, is_label=False):
    """
    Make a binary data file for learning.
    """
    binary_fname = fname.replace('.txt', '.bin')
    if os.path.exists(binary_fname): return

    with open(fname, 'r', encoding='utf8') as f:
        ls = f.readlines()
        data = [line.strip() for line in ls]

    assert all(['|||' in dat for dat in data]) if is_label else all(['|||'  not in dat for dat in data])

    with open(binary_fname, 'wb') as f:
        for line in data:
            if is_label:
                split_line = line.split('|||')
                assert len(split_line) == 2
                label, text = split_line[0].strip(), split_line[1].strip()
            else:
                text = line
            example = example_pb2.Example()
            example.features.feature['text'].bytes_list.value.extend([text.encode()])
            if is_label:
                example.features.feature['label'].bytes_list.value.extend([label.encode()])
            example_str = example.SerializeToString()
            str_len = len(example_str)
            f.write(struct.pack('q', str_len))
            f.write(struct.pack('%ds' % str_len, example_str))
    return

def main():
    train_fname = './data/datasets/{}/train.txt'.format(args.dataset)
    test_fname = './data/datasets/{}/test.txt'.format(args.dataset)
    test_label_fname = './data/datasets/{}/test_label.txt'.format(args.dataset)
    vocab_fname = './data/vocab.txt'
    vocab_size = args.vocab_size

    parsed_train_fname = tokenize_train_file(train_fname)
    label_map, parsed_test_fname = tokenize_labeled_test_file(test_fname, test_label_fname)
    build_vocab(parsed_train_fname, vocab_fname, vocab_size=vocab_size)

    make_binary_dataset(parsed_train_fname, False)
    make_binary_dataset(parsed_test_fname, True)

if __name__ == '__main__':
    main()


from pykit import logutil

print logutil.get_root_log_fn()


#!/usr/bin/env python3

from os import environ
from curio import Channel, run
syncword = environ.get('RRIDBOT_SYNC')
chan = ('localhost', 12345)
async def consumer():
    ch = Channel(chan)
    c = await ch.accept(authkey=syncword.encode())
    myset = set()
    while True:
        try:
            msg = await c.recv()
        except (EOFError, ConnectionResetError) as e:  # in the event that the client closes
            print('resetting')
            myset = set()
            c = await ch.accept(authkey=syncword.encode())
            continue
        if msg is None:  # explicit reset
            myset = set()
        else:
            op, uri = msg.split(' ', 1)
            print(op, uri)
            if op == 'add':
                if uri in myset:
                    await c.send(True)
                else:
                    myset.add(uri)
                    await c.send(False)
            elif op == 'del':
                myset.discard(uri)
                await c.send(False)
            else:
                await c.send('ERROR')
        print(myset)

if __name__ == '__main__':
    run(consumer)


from setuptools import setup

setup(
    name = 'PyXiaomiGateway',
    packages = ['xiaomi_gateway'],
    install_requires=['cryptography>=2.1.1'],
    version = '0.11.1',
    description = 'A library to communicate with the Xiaomi Gateway',
    author='Daniel Hoyer Iversen',
    url='https://github.com/Danielhiversen/PyXiaomiGateway/',
    classifiers=[
        'Development Status :: 3 - Alpha',
        'Environment :: Other Environment',
        'Intended Audience :: Developers',
        'Operating System :: OS Independent',
        'Programming Language :: Python',
        'Topic :: Home Automation',
        'Topic :: Software Development :: Libraries :: Python Modules'
        ]
)


from nose.tools import *  # noqa

from framework.auth.core import Auth

from tests.base import OsfTestCase
from tests.factories import AuthUserFactory, ProjectFactory, BookmarkCollectionFactory

from scripts.analytics.addon_snapshot import AddonSnapshot

from website.models import Node
from framework.auth.core import User
from website.settings import ADDONS_AVAILABLE
from addons.github.tests.factories import GitHubAccountFactory
from addons.github.model import GitHubNodeSettings, GitHubUserSettings
from addons.googledrive.tests.factories import GoogleDriveAccountFactory
from addons.googledrive.model import GoogleDriveNodeSettings, GoogleDriveUserSettings

class TestAddonCount(OsfTestCase):
    def setUp(self):
        super(TestAddonCount, self).setUp()
        self.user = AuthUserFactory()
        self.node = ProjectFactory(creator=self.user)
        self.user.add_addon('github')
        self.user_addon = self.user.get_addon('github')

        self.external_account = GitHubAccountFactory(display_name='hmoco1')

        self.user_settings = self.user.get_or_add_addon('github')

        self.user_settings.save()
        self.user.external_accounts.append(self.external_account)
        self.user.save()
        self.node.add_addon('github', Auth(self.user))
        self.node_addon = self.node.get_addon('github')
        self.node_addon.user = self.user.fullname
        self.node_addon.repo = '29 #Strafford APTS'
        self.node_addon.user_settings = self.user_addon
        self.node_addon.external_account = self.external_account
        self.node_addon.save()

        self.user_settings.grant_oauth_access(
            node=self.node,
            external_account=self.external_account,
        )
    def tearDown(self):
        GitHubNodeSettings.remove()
        GitHubUserSettings.remove()
        GoogleDriveNodeSettings.remove()
        GoogleDriveUserSettings.remove()

    def test_run_for_all_addon(self):
        results = AddonSnapshot().get_events()
        names = [res['provider']['name'] for res in results]
        for addon in ADDONS_AVAILABLE:
            assert_in(addon.short_name, names)

    def test_one_user_one_node_one_addon(self):
        results = AddonSnapshot().get_events()
        github_res = [res for res in results if res['provider']['name'] == 'github'][0]
        assert_equal(github_res['users']['enabled'], 1)
        assert_equal(github_res['nodes']['total'], 1)

    def test_one_user_one_node_one_addon_one_node_linked(self):
        results = AddonSnapshot().get_events()
        github_res = [res for res in results if res['provider']['name'] == 'github'][0]
        assert_equal(github_res['users']['enabled'], 1)
        assert_equal(github_res['nodes']['total'], 1)

    def test_one_user_with_multiple_githubs(self):
        oauth_settings2 = GitHubAccountFactory(display_name='hmoco2')
        oauth_settings2.save()
        self.user.external_accounts.append(oauth_settings2)
        self.user.save()
        results = AddonSnapshot().get_events()
        github_res = [res for res in results if res['provider']['name'] == 'github'][0]
        assert_equal(github_res['users']['enabled'], 1)

    def test_one_user_with_multiple_addons(self):
        results = AddonSnapshot().get_events()
        github_res = [res for res in results if res['provider']['name'] == 'github'][0]
        googledrive_res = [res for res in results if res['provider']['name'] == 'googledrive'][0]
        assert_equal(github_res['users']['enabled'], 1)
        assert_equal(googledrive_res['users']['enabled'], 0)

        self.user.add_addon('googledrive')
        oauth_settings = GoogleDriveAccountFactory()
        oauth_settings.save()
        self.user.external_accounts.append(oauth_settings)
        self.user.save()
        results = AddonSnapshot().get_events()
        github_res = [res for res in results if res['provider']['name'] == 'github'][0]
        googledrive_res = [res for res in results if res['provider']['name'] == 'googledrive'][0]
        assert_equal(github_res['users']['enabled'], 1)
        assert_equal(googledrive_res['users']['enabled'], 1)

    def test_many_users_each_with_a_different_github(self):
        user = AuthUserFactory()
        user.add_addon('github')
        oauth_settings2 = GitHubAccountFactory(display_name='hmoco2')
        oauth_settings2.save()
        user.external_accounts.append(oauth_settings2)
        user.save()
        results = AddonSnapshot().get_events()
        github_res = [res for res in results if res['provider']['name'] == 'github'][0]
        assert_equal(github_res['users']['enabled'], 2)
        assert_equal(github_res['users']['authorized'], 1)
        assert_equal(github_res['users']['linked'], 1)

    def test_many_users_each_with_the_same_github_enabled(self):
        user = AuthUserFactory()
        user.add_addon('github')
        user.external_accounts.append(self.external_account)
        user.save()
        results = AddonSnapshot().get_events()
        github_res = [res for res in results if res['provider']['name'] == 'github'][0]
        assert_equal(github_res['users']['enabled'], 2)

    def test_github_enabled_not_linked_or_authorized(self):
        user = AuthUserFactory()
        user.add_addon('github')
        user.external_accounts.append(self.external_account)
        user.save()
        results = AddonSnapshot().get_events()
        github_res = [res for res in results if res['provider']['name'] == 'github'][0]
        assert_equal(github_res['users']['enabled'], 2)
        assert_equal(github_res['users']['authorized'], 1)
        assert_equal(github_res['users']['linked'], 1)

    def test_one_node_with_multiple_addons(self):
        results = AddonSnapshot().get_events()
        github_res = [res for res in results if res['provider']['name'] == 'github'][0]
        googledrive_res = [res for res in results if res['provider']['name'] == 'googledrive'][0]
        assert_equal(github_res['nodes']['total'], 1)
        assert_equal(googledrive_res['nodes']['total'], 0)

        self.user.add_addon('googledrive')
        user_addon = self.user.get_addon('googledrive')
        oauth_settings = GoogleDriveAccountFactory()
        oauth_settings.save()
        self.user.external_accounts.append(oauth_settings)
        self.user.save()
        self.node.add_addon('googledrive', Auth(self.user))
        node_addon = self.node.get_addon('googledrive')
        node_addon.user = self.user.fullname
        node_addon.user_settings = user_addon
        node_addon.external_account = oauth_settings
        node_addon.save()
        results = AddonSnapshot().get_events()
        github_res = [res for res in results if res['provider']['name'] == 'github'][0]
        googledrive_res = [res for res in results if res['provider']['name'] == 'googledrive'][0]
        assert_equal(github_res['nodes']['total'], 1)
        assert_equal(googledrive_res['nodes']['total'], 1)

    def test_many_nodes_with_one_addon(self):
        results = AddonSnapshot().get_events()
        github_res = [res for res in results if res['provider']['name'] == 'github'][0]
        assert_equal(github_res['nodes']['total'], 1)

        node = ProjectFactory(creator=self.user)
        node.add_addon('github', Auth(self.user))
        node_addon = node.get_addon('github')
        node_addon.user = self.user.fullname
        node_addon.repo = '8 (circle)'
        node_addon.user_settings = self.user_addon
        node_addon.external_account = self.external_account
        node_addon.save()
        node.save()

        results = AddonSnapshot().get_events()
        github_res = [res for res in results if res['provider']['name'] == 'github'][0]
        assert_equal(github_res['nodes']['total'], 2)

    def test_node_count_deleted_addon(self):
        results = AddonSnapshot().get_events()
        github_res = [res for res in results if res['provider']['name'] == 'github'][0]
        assert_equal(github_res['nodes']['deleted'], 0)

        node = ProjectFactory(creator=self.user)
        node.add_addon('github', Auth(self.user))
        node_addon = node.get_addon('github')
        node_addon.delete()

        results = AddonSnapshot().get_events()
        github_res = [res for res in results if res['provider']['name'] == 'github'][0]
        assert_equal(github_res['nodes']['deleted'], 1)

    def test_node_count_disconected_addon(self):
        results = AddonSnapshot().get_events()
        github_res = [res for res in results if res['provider']['name'] == 'github'][0]
        assert_equal(github_res['nodes']['disconnected'], 0)

        node = ProjectFactory(creator=self.user)
        node.add_addon('github', Auth(self.user))
        node_addon = node.get_addon('github')
        node_addon.external_account = None
        node_addon.save()

        results = AddonSnapshot().get_events()
        github_res = [res for res in results if res['provider']['name'] == 'github'][0]
        assert_equal(github_res['nodes']['disconnected'], 1)

    def test_all_users_have_wiki_osfstorage_enabled(self):
        all_user_count = User.find().count()
        results = AddonSnapshot().get_events()
        osfstorage_res = [res for res in results if res['provider']['name'] == 'osfstorage'][0]
        wiki_res = [res for res in results if res['provider']['name'] == 'osfstorage'][0]

        assert_equal(osfstorage_res['users']['enabled'], all_user_count)
        assert_equal(wiki_res['users']['enabled'], all_user_count)

    def test_wiki_deleted_shows_as_deleted(self):
        node = ProjectFactory(creator=self.user)
        node.delete_addon('wiki', auth=Auth(self.user))

        results = AddonSnapshot().get_events()
        wiki_res = [res for res in results if res['provider']['name'] == 'wiki'][0]

        assert_equal(wiki_res['nodes']['deleted'], 1)

    def test_node_settings_has_no_owner_not_connected(self):
        self.node_addon.owner = None
        self.node_addon.save()

        results = AddonSnapshot().get_events()
        storage_res = [res for res in results if res['provider']['name'] == 'github'][0]
        assert_equal(storage_res['nodes']['connected'], 0)

    def test_bookmark_collection_not_counted(self):
        BookmarkCollectionFactory(creator=self.user)
        all_node_count = Node.find().count()

        results = AddonSnapshot().get_events()
        storage_res = [res for res in results if res['provider']['name'] == 'osfstorage'][0]
        assert_equal(storage_res['nodes']['connected'], all_node_count - 1)


import aiohttp_csrf
import pytest
from aiohttp import web

SESSION_NAME = COOKIE_NAME = 'csrf_token'
FORM_FIELD_NAME = HEADER_NAME = 'X-CSRF-TOKEN'

@pytest.yield_fixture
def init_app():
    def go(
        loop,
        policy,
        storage,
        handlers,
        error_renderer=None,
    ):
        app = web.Application()

        kwargs = {
            'policy': policy,
            'storage': storage,
        }

        if error_renderer is not None:
            kwargs['error_renderer'] = error_renderer

        aiohttp_csrf.setup(app, **kwargs)

        for method, url, handler in handlers:
            app.router.add_route(
                method,
                url,
                handler,
            )

        return app

    yield go

@pytest.fixture(params=[
    (aiohttp_csrf.policy.FormPolicy, (FORM_FIELD_NAME,)),
    (aiohttp_csrf.policy.FormAndHeaderPolicy, (HEADER_NAME, FORM_FIELD_NAME)),
])
def csrf_form_policy(request):
    _class, args = request.param

    return _class(*args)

@pytest.fixture(params=[
    (aiohttp_csrf.policy.HeaderPolicy, (HEADER_NAME,)),
    (aiohttp_csrf.policy.FormAndHeaderPolicy, (HEADER_NAME, FORM_FIELD_NAME)),
])
def csrf_header_policy(request):
    _class, args = request.param

    return _class(*args)

@pytest.fixture(params=[
    (aiohttp_csrf.storage.SessionStorage, (SESSION_NAME,)),
    (aiohttp_csrf.storage.CookieStorage, (COOKIE_NAME,)),
])
def csrf_storage(request):
    _class, args = request.param

    return _class(*args)


#!/home/access/Downloads/Desktop/Django_IPs/week4_IP/neighbourhood/virtual/bin/python3.8
# When the django-admin.py deprecation ends, remove this script.
import warnings

from django.core import management

try:
    from django.utils.deprecation import RemovedInDjango40Warning
except ImportError:
    raise ImportError(
        'django-admin.py was deprecated in Django 3.1 and removed in Django '
        '4.0. Please manually remove this script from your virtual environment '
        'and use django-admin instead.'
    )

if __name__ == "__main__":
    warnings.warn(
        'django-admin.py is deprecated in favor of django-admin.',
        RemovedInDjango40Warning,
    )
    management.execute_from_command_line()


"""
    sphinxcontrib.openapi.openapi30
    -------------------------------

    The OpenAPI 3.0.0 spec renderer. Based on ``sphinxcontrib-httpdomain``.

    :copyright: (c) 2016, Ihor Kalnytskyi.
    :license: BSD, see LICENSE for details.
"""

import copy

import collections
import collections.abc

from datetime import datetime
import itertools
import json
import re
from urllib import parse
from http.client import responses as http_status_codes

from sphinx.util import logging

from sphinxcontrib.openapi import utils

LOG = logging.getLogger(__name__)

# https://github.com/OAI/OpenAPI-Specification/blob/3.0.2/versions/3.0.0.md#data-types
_TYPE_MAPPING = {
    ('integer', 'int32'): 1,  # integer
    ('integer', 'int64'): 1,  # long
    ('number', 'float'): 1.0,  # float
    ('number', 'double'): 1.0,  # double
    ('boolean', None): True,  # boolean
    ('string', None): 'string',  # string
    ('string', 'byte'): 'c3RyaW5n',  # b'string' encoded in base64,  # byte
    ('string', 'binary'): '01010101',  # binary
    ('string', 'date'): datetime.now().date().isoformat(),  # date
    ('string', 'date-time'): datetime.now().isoformat(),  # dateTime
    ('string', 'password'): '********',  # password

    # custom extensions to handle common formats
    ('string', 'email'): 'name@example.com',
    ('string', 'zip-code'): '90210',
    ('string', 'uri'): 'https://example.com',

    # additional fallthrough cases
    ('integer', None): 1,  # integer
    ('number', None): 1.0,  # <fallthrough>
}

_READONLY_PROPERTY = object()  # sentinel for values not included in requests

def _dict_merge(dct, merge_dct):
    """Recursive dict merge.

    Inspired by :meth:``dict.update()``, instead of updating only top-level
    keys, dict_merge recurses down into dicts nested to an arbitrary depth,
    updating keys. The ``merge_dct`` is merged into ``dct``.

    From https://gist.github.com/angstwad/bf22d1822c38a92ec0a9

    Arguments:
        dct: dict onto which the merge is executed
        merge_dct: dct merged into dct
    """
    for k in merge_dct.keys():
        if (k in dct and isinstance(dct[k], dict)
                and isinstance(merge_dct[k], collections.abc.Mapping)):
            _dict_merge(dct[k], merge_dct[k])
        else:
            dct[k] = merge_dct[k]

def _parse_schema(schema, method):
    """
    Convert a Schema Object to a Python object.

    Args:
        schema: An ``OrderedDict`` representing the schema object.
    """
    if method and schema.get('readOnly', False):
        return _READONLY_PROPERTY

    # allOf: Must be valid against all of the subschemas
    if 'allOf' in schema:
        schema_ = copy.deepcopy(schema['allOf'][0])
        for x in schema['allOf'][1:]:
            _dict_merge(schema_, x)

        return _parse_schema(schema_, method)

    # anyOf: Must be valid against any of the subschemas
    # TODO(stephenfin): Handle anyOf

    # oneOf: Must be valid against exactly one of the subschemas
    if 'oneOf' in schema:
        # we only show the first one since we can't show everything
        return _parse_schema(schema['oneOf'][0], method)

    if 'enum' in schema:
        # we only show the first one since we can't show everything
        return schema['enum'][0]

    schema_type = schema.get('type', 'object')

    if schema_type == 'array':
        # special case oneOf and anyOf so that we can show examples for all
        # possible combinations
        if 'oneOf' in schema['items']:
            return [
                _parse_schema(x, method) for x in schema['items']['oneOf']
            ]

        if 'anyOf' in schema['items']:
            return [
                _parse_schema(x, method) for x in schema['items']['anyOf']
            ]

        return [_parse_schema(schema['items'], method)]

    if schema_type == 'object':
        if method and 'properties' in schema and \
                all(v.get('readOnly', False)
                    for v in schema['properties'].values()):
            return _READONLY_PROPERTY

        results = []
        for name, prop in schema.get('properties', {}).items():
            result = _parse_schema(prop, method)
            if result != _READONLY_PROPERTY:
                results.append((name, result))

        return collections.OrderedDict(results)

    if (schema_type, schema.get('format')) in _TYPE_MAPPING:
        return _TYPE_MAPPING[(schema_type, schema.get('format'))]

    return _TYPE_MAPPING[(schema_type, None)]  # unrecognized format

def _example(media_type_objects, method=None, endpoint=None, status=None,
             nb_indent=0):
    """
    Format examples in `Media Type Object` openapi v3 to HTTP request or
    HTTP response example.
    If method and endpoint is provided, this function prints a request example
    else status should be provided to print a response example.

    Arguments:
        media_type_objects (Dict[str, Dict]): Dict containing
            Media Type Objects.
        method: The HTTP method to use in example.
        endpoint: The HTTP route to use in example.
        status: The HTTP status to use in example.
    """
    indent = '   '
    extra_indent = indent * nb_indent

    if method is not None:
        method = method.upper()
    else:
        try:
            # one of possible values for status might be 'default'.
            # in the case, just fallback to '-'
            status_text = http_status_codes[int(status)]
        except (ValueError, KeyError):
            status_text = '-'

    # Provide request samples for GET requests
    if method == 'GET':
        media_type_objects[''] = {
            'examples': {'Example request': {'value': ''}}}

    for content_type, content in media_type_objects.items():
        examples = content.get('examples')
        example = content.get('example')

        # Try to get the example from the schema
        if example is None and 'schema' in content:
            example = content['schema'].get('example')

        if examples is None:
            examples = {}
            if not example:
                if re.match(r"application/[a-zA-Z\+]*json", content_type) is \
                        None:
                    LOG.info('skipping non-JSON example generation.')
                    continue
                example = _parse_schema(content['schema'], method=method)

            if method is None:
                examples['Example response'] = {
                    'value': example,
                }
            else:
                examples['Example request'] = {
                    'value': example,
                }

        for example in examples.values():
            # According to OpenAPI v3 specs, string examples should be left unchanged
            if not isinstance(example['value'], str):
                example['value'] = json.dumps(
                    example['value'], indent=4, separators=(',', ': '))

        for example_name, example in examples.items():
            if 'summary' in example:
                example_title = '{example_name} - {example[summary]}'.format(
                    **locals())
            else:
                example_title = example_name

            yield ''
            yield '{extra_indent}**{example_title}:**'.format(**locals())
            yield ''
            yield '{extra_indent}.. sourcecode:: http'.format(**locals())
            yield ''

            # Print http request example
            if method:
                yield '{extra_indent}{indent}{method} {endpoint} HTTP/1.1' \
                    .format(**locals())
                yield '{extra_indent}{indent}Host: example.com' \
                    .format(**locals())
                if content_type:
                    yield '{extra_indent}{indent}Content-Type: {content_type}'\
                        .format(**locals())

            # Print http response example
            else:
                yield '{extra_indent}{indent}HTTP/1.1 {status} {status_text}' \
                    .format(**locals())
                yield '{extra_indent}{indent}Content-Type: {content_type}' \
                    .format(**locals())

            yield ''
            for example_line in example['value'].splitlines():
                yield '{extra_indent}{indent}{example_line}'.format(**locals())
            if example['value'].splitlines():
                yield ''

def convert_json_schema(schema, directive=':<json'):
    """
    Convert json schema to `:<json` sphinx httpdomain.
    """

    output = []

    def _convert(schema, name='', required=False):
        """
        Fill the output list, with 2-tuple (name, template)
        i.e: ('user.age', 'str user.age: the age of user')
        This allow to sort output by field name
        """

        type_ = schema.get('type', 'any')
        required_properties = schema.get('required', ())
        if type_ == 'object' and schema.get('properties'):
            for prop, next_schema in schema.get('properties', {}).items():
                _convert(
                    next_schema, '{name}.{prop}'.format(**locals()),
                    (prop in required_properties))

        elif type_ == 'array':
            _convert(schema['items'], name + '[]')

        else:
            if name:
                name = name.lstrip('.')
                constraints = []
                if required:
                    constraints.append('required')
                if schema.get('readOnly', False):
                    constraints.append('read only')
                if constraints:
                    constraints = '({})'.format(', '.join(constraints))
                else:
                    constraints = ''

                if schema.get('description', ''):
                    if constraints:
                        output.append((
                            name,
                            '{type_} {name}:'
                            ' {schema[description]}'
                            ' {constraints}'.format(**locals())))
                    else:
                        output.append((
                            name,
                            '{type_} {name}:'
                            ' {schema[description]}'.format(**locals())))

                else:
                    if constraints:
                        output.append(
                            (name,
                             '{type_} {name}:'
                             ' {constraints}'.format(**locals())))
                    else:
                        output.append(
                            (name,
                             '{type_} {name}:'.format(**locals())))

    _convert(schema)

    for _, render in sorted(output):
        yield '{} {}'.format(directive, render)

def _httpresource(endpoint, method, properties, convert, render_examples,
                  render_request):
    # https://github.com/OAI/OpenAPI-Specification/blob/3.0.2/versions/3.0.0.md#operation-object
    parameters = properties.get('parameters', [])
    responses = properties['responses']
    query_param_examples = []
    indent = '   '

    yield '.. http:{0}:: {1}'.format(method, endpoint)
    yield '   :synopsis: {0}'.format(properties.get('summary', 'null'))
    yield ''

    if 'summary' in properties:
        for line in properties['summary'].splitlines():
            yield '{indent}**{line}**'.format(**locals())
        yield ''

    if 'description' in properties:
        for line in convert(properties['description']).splitlines():
            yield '{indent}{line}'.format(**locals())
        yield ''

    # print request's path params
    for param in filter(lambda p: p['in'] == 'path', parameters):
        yield indent + ':param {type} {name}:'.format(
            type=param['schema']['type'],
            name=param['name'])

        for line in convert(param.get('description', '')).splitlines():
            yield '{indent}{indent}{line}'.format(**locals())

    # print request's query params
    for param in filter(lambda p: p['in'] == 'query', parameters):
        yield indent + ':query {type} {name}:'.format(
            type=param['schema']['type'],
            name=param['name'])
        for line in convert(param.get('description', '')).splitlines():
            yield '{indent}{indent}{line}'.format(**locals())
        if param.get('required', False):
            yield '{indent}{indent}(Required)'.format(**locals())
            example = _parse_schema(param['schema'], method)
            example = param.get('example', example)
            if param.get('explode', False) and isinstance(example, list):
                for v in example:
                    query_param_examples.append((param['name'], v))
            elif param.get('explode', False) and isinstance(example, dict):
                for k, v in example.items():
                    query_param_examples.append((k, v))
            else:
                query_param_examples.append((param['name'], example))

    # print request content
    if render_request:
        request_content = properties.get('requestBody', {}).get('content', {})
        if request_content and 'application/json' in request_content:
            schema = request_content['application/json']['schema']
            yield ''
            for line in convert_json_schema(schema):
                yield '{indent}{line}'.format(**locals())
            yield ''
            req_properties = json.dumps(schema.get('properties', {}), indent=2,
                                        separators=(',', ':'))
            yield '{indent}**Request body:**'.format(**locals())
            yield ''
            yield '{indent}.. sourcecode:: json'.format(**locals())
            yield ''
            for line in req_properties.splitlines():
                # yield indent + line
                yield '{indent}{indent}{line}'.format(**locals())
                # yield ''

    # print request example
    if render_examples:
        endpoint_examples = endpoint
        if query_param_examples:
            endpoint_examples = endpoint + "?" + \
                parse.urlencode(query_param_examples)

        # print request example
        request_content = properties.get('requestBody', {}).get('content', {})
        for line in _example(
                request_content,
                method,
                endpoint=endpoint_examples,
                nb_indent=1):
            yield line

    # print response status codes
    for status, response in responses.items():
        yield '{indent}:status {status}:'.format(**locals())
        for line in convert(response['description']).splitlines():
            yield '{indent}{indent}{line}'.format(**locals())

        # print response example
        if render_examples:
            for line in _example(
                    response.get('content', {}), status=status, nb_indent=2):
                yield line

    # print request header params
    for param in filter(lambda p: p['in'] == 'header', parameters):
        yield indent + ':reqheader {name}:'.format(**param)
        for line in convert(param.get('description', '')).splitlines():
            yield '{indent}{indent}{line}'.format(**locals())
        if param.get('required', False):
            yield '{indent}{indent}(Required)'.format(**locals())

    # print response headers
    for status, response in responses.items():
        for headername, header in response.get('headers', {}).items():
            yield indent + ':resheader {name}:'.format(name=headername)
            for line in convert(header['description']).splitlines():
                yield '{indent}{indent}{line}'.format(**locals())

    for cb_name, cb_specs in properties.get('callbacks', {}).items():
        yield ''
        yield indent + '.. admonition:: Callback: ' + cb_name
        yield ''

        for cb_endpoint in cb_specs.keys():
            for cb_method, cb_properties in cb_specs[cb_endpoint].items():
                for line in _httpresource(
                        cb_endpoint,
                        cb_method,
                        cb_properties,
                        convert=convert,
                        render_examples=render_examples,
                        render_request=render_request):
                    if line:
                        yield indent+indent+line
                    else:
                        yield ''

    yield ''

def _header(title):
    yield title
    yield '=' * len(title)
    yield ''

def openapihttpdomain(spec, **options):
    generators = []

    # OpenAPI spec may contain JSON references, common properties, etc.
    # Trying to render the spec "As Is" will require to put multiple
    # if-s around the code. In order to simplify flow, let's make the
    # spec to have only one (expected) schema, i.e. normalize it.
    utils.normalize_spec(spec, **options)

    # Paths list to be processed
    paths = []

    # If 'paths' are passed we've got to ensure they exist within an OpenAPI
    # spec; otherwise raise error and ask user to fix that.
    if 'paths' in options:
        if not set(options['paths']).issubset(spec['paths']):
            raise ValueError(
                'One or more paths are not defined in the spec: %s.' % (
                    ', '.join(set(options['paths']) - set(spec['paths'])),
                )
            )
        paths = options['paths']

    # Check against regular expressions to be included
    if 'include' in options:
        for i in options['include']:
            ir = re.compile(i)
            for path in spec['paths']:
                if ir.match(path):
                    paths.append(path)

    # If no include nor paths option, then take full path
    if 'include' not in options and 'paths' not in options:
        paths = spec['paths']

    # Remove paths matching regexp
    if 'exclude' in options:
        _paths = []
        for e in options['exclude']:
            er = re.compile(e)
            for path in paths:
                if not er.match(path):
                    _paths.append(path)
        paths = _paths

    render_request = False
    if 'request' in options:
        render_request = True

    convert = utils.get_text_converter(options)

    # https://github.com/OAI/OpenAPI-Specification/blob/3.0.2/versions/3.0.0.md#paths-object
    if 'group' in options:
        groups = collections.OrderedDict(
            [(x['name'], []) for x in spec.get('tags', {})]
            )

        for endpoint in paths:
            for method, properties in spec['paths'][endpoint].items():
                key = properties.get('tags', [''])[0]
                groups.setdefault(key, []).append(_httpresource(
                    endpoint,
                    method,
                    properties,
                    convert,
                    render_examples='examples' in options,
                    render_request=render_request))

        for key in groups.keys():
            if key:
                generators.append(_header(key))
            else:
                generators.append(_header('default'))

            generators.extend(groups[key])
    else:
        for endpoint in paths:
            for method, properties in spec['paths'][endpoint].items():
                generators.append(_httpresource(
                    endpoint,
                    method,
                    properties,
                    convert,
                    render_examples='examples' in options,
                    render_request=render_request))

    return iter(itertools.chain(*generators))


from django.core.mail import EmailMultiAlternatives
from django.template.loader import render_to_string

def send_email(name,receiver):
    # Creating message subject and sender
    subject = 'Welcome to Our Neighbourhood'
    sender = 'emmahg6@gmail.com'

    #passing in the context vairables
    text_content = render_to_string('email/notification.txt',{"name": name})
    html_content = render_to_string('email/notification.html',{"name": name})

    msg = EmailMultiAlternatives(subject,text_content,sender,[receiver])
    msg.attach_alternative(html_content,'text/html')
    msg.send()


"""Project: Eskapade - A python-based package for data analysis.

Module: spark_analysis.data_conversion

Created: 2017/05/30

Description:
    Converters between Spark, Pandas, and Python data formats

Authors:
    KPMG Advanced Analytics & Big Data team, Amstelveen, The Netherlands

Redistribution and use in source and binary forms, with or without
modification, are permitted according to the terms listed in the file
LICENSE.
"""

import uuid

import pyspark

from eskapadespark.helpers import apply_transform_funcs
from eskapade.logger import Logger

SPARK_SQL_TYPES = pyspark.sql.types._type_mappings
logger = Logger()

def create_spark_df(spark, data, schema=None, process_methods=None, **kwargs):
    """Create a Spark data frame from data in a different format.

    A Spark data frame is created with either a specified schema or a schema
    inferred from the input data.  The schema can be specified with the
    keyword argument "schema".

    Functions to transform the data frame after creation can be specified by
    the keyword argument "process_methods".  The value of this argument is
    an iterable of (function, arguments, keyword arguments) tuples to apply.

    The data frame is created with the createDataFrame function of the
    SparkSession.  Remaining keyword arguments are passed to this function.

    >>> spark = pyspark.sql.SparkSession.builder.getOrCreate()
    >>> df = create_spark_df(spark,
    >>>                      [[1, 1.1, 'one'], [2, 2.2, 'two']],
    >>>                      schema=['int', 'float', 'str'],
    >>>                      process_methods=[('repartition', (), {'numPartitions': 6})])
    >>> df.show()
    +---+-----+---+
    |int|float|str|
    +---+-----+---+
    |  2|  2.2|two|
    |  1|  1.1|one|
    +---+-----+---+

    :param pyspark.sql.SparkSession spark: SparkSession instance
    :param data: input dataset
    :param schema: schema of created data frame
    :param iterable process_methods: methods to apply on the data frame after creation
    :returns: created data frame
    :rtype: pyspark.sql.DataFrame
    """
    # check if data-frame schema was provided
    if isinstance(schema, int):
        # infer schema from a single row (prevents Spark >= 1.6.1 from checking schema of all rows)
        def get_row(data, ind):
            """Get row."""
            try:
                return data.iloc[ind].tolist()
            except AttributeError:
                pass
            try:
                row = data.first()
                if ind > 0:
                    logger.warning('Inferring data-frame schema from first row, instead of row with index {i:d}', i=ind)
                return row
            except AttributeError:
                pass
            try:
                return data[ind]
            except TypeError:
                raise TypeError('Unable to get row from data of type "{!s}" to infer schema.'.format(type(data)))

        row = get_row(data, schema)

        def to_python_type(var):
            """Get item."""
            try:
                return var.item()
            except AttributeError:
                return var

        schema = pyspark.sql.types._infer_schema(tuple(to_python_type(it) for it in row))
        try:
            for t, n in zip(schema.fields, data.columns):
                t.name = str(n)
        except AttributeError:
            pass
    elif isinstance(schema, dict):
        # create schema from dictionary of (name, data type) pairs
        schema = df_schema(schema)
    kwargs['schema'] = schema

    # check if input is a data frame
    if isinstance(data, pyspark.sql.DataFrame):
        if not kwargs['schema']:
            kwargs['schema'] = data.schema
        data = data.rdd

    # create and transform data frame
    df = spark.createDataFrame(data, **kwargs)
    if process_methods:
        df = apply_transform_funcs(df, process_methods)
    return df

def df_schema(schema_spec):
    """Create Spark data-frame schema.

    Create a schema for a Spark data frame from a dictionary of (name, data
    type) pairs, describing the columns.  Data types are specified by Python
    types or by Spark-SQL types from the pyspark.sql.types module.

    >>> from collections import OrderedDict as odict
    >>> schema_dict = odict()
    >>> schema_dict['foo'] = pyspark.sql.types.IntegerType()
    >>> schema_dict['bar'] = odict([('descr', str), ('val', float)])
    >>> print(schema_dict)
    OrderedDict([('foo', IntegerType), ('bar', OrderedDict([('descr', <class 'str'>), ('val', <class 'float'>)]))])
    >>> spark = pyspark.sql.SparkSession.builder.getOrCreate()
    >>> df = spark.createDataFrame([(1, ('one', 1.1)), (2, ('two', 2.2))], schema=df_schema(schema_dict))
    >>> df.show()
    +---+---------+
    |foo|      bar|
    +---+---------+
    |  1|[one,1.1]|
    |  2|[two,2.2]|
    +---+---------+

    :param dict schema_spec: schema specification
    :returns: data-frame schema
    :rtype: pyspark.sql.types.StructType
    :raises: TypeError if data type is specified incorrectly
    """
    def get_field(name, data_type):
        """Return a struct field for specified data type."""
        # treat dictionaries as struct types
        if isinstance(data_type, dict):
            data_type = pyspark.sql.types.StructType([get_field(*spec) for spec in data_type.items()])

        # convert Python types to Spark-SQL types
        data_type = SPARK_SQL_TYPES.get(data_type, data_type)

        # convert Spark-SQL type classes to Spark-SQL types
        if isinstance(data_type, type) and issubclass(data_type, pyspark.sql.types.DataType):
            data_type = data_type()

        # check and return data type
        if not isinstance(data_type, pyspark.sql.types.DataType):
            raise TypeError('Type specifications for data-frame schemas must be DataTypes or dictionaries')
        return pyspark.sql.types.StructField(str(name), data_type)

    # return a struct type with a list of struct fields for specified data types
    return pyspark.sql.types.StructType([get_field(*spec) for spec in schema_spec.items()])

def hive_table_from_df(spark, df, db, table):
    """Create a Hive table from a Spark data frame.

    :param pyspark.sql.SparkSession spark: SparkSession instance
    :param pyspark.sql.DataFrame df: input data frame
    :param str db: database for table
    :param str table: name of table
    """
    # register temporary table
    temp_name = '{0:s}_{1:s}'.format(table, uuid.uuid4().hex)
    df.createOrReplaceTempView(temp_name)

    # create table
    table_spec = '.'.join(s for s in (db, table) if s)
    create_table_query = 'CREATE TABLE {spec} AS SELECT {cols} FROM {name}'\
        .format(name=temp_name, spec=table_spec, cols=', '.join(c for c in df.columns))
    logger.debug(create_table_query)
    spark.sql(create_table_query)