session = boto3.session.Session(<EOL>profile_name=aws_profile,<EOL>aws_access_key_id=aws_access_key_id,<EOL>aws_secret_access_key=aws_secret_access_key)<EOL>s3 = session.resource('<STR_LIT>')<EOL>bucket = s3.Bucket(bucket_name)<EOL>return bucket<EOL>

Open an S3 Bucket resource.

    Parameters
    ----------
    bucket_name : `str`
        Name of the S3 bucket.
    aws_access_key_id : `str`, optional
        The access key for your AWS account. Also set
        ``aws_secret_access_key``.
    aws_secret_access_key : `str`, optional
        The secret key for your AWS account.
    aws_profile : `str`, optional
        Name of AWS profile in :file:`~/.aws/credentials`. Use this instead
        of ``aws_access_key_id`` and ``aws_secret_access_key`` for file-based
        credentials.

    Returns
    -------
    bucket : Boto3 S3 Bucket instance
        The S3 bucket as a Boto3 instance.


logger = logging.getLogger(__name__)<EOL>if skip_upload:<EOL><INDENT>click.echo('<STR_LIT>')<EOL>sys.exit(<NUM_LIT:0>)<EOL><DEDENT>logger.debug('<STR_LIT>', ci_env)<EOL>logger.debug('<STR_LIT>'<EOL>'<STR_LIT>',<EOL>on_travis_push, on_travis_pr, on_travis_api, on_travis_cron)<EOL>if ci_env == '<STR_LIT>' and_should_skip_travis_event(<EOL>on_travis_push, on_travis_pr, on_travis_api, on_travis_cron):<EOL><INDENT>sys.exit(<NUM_LIT:0>)<EOL><DEDENT>ensure_login(ctx)<EOL>git_refs = _get_git_refs(ci_env, git_ref)<EOL>build_resource = register_build(<EOL>ctx.obj['<STR_LIT>'],<EOL>ctx.obj['<STR_LIT>'],<EOL>product,<EOL>git_refs<EOL>)<EOL>logger.debug('<STR_LIT>', build_resource)<EOL>upload_dir(<EOL>build_resource['<STR_LIT>'],<EOL>build_resource['<STR_LIT>'],<EOL>dirname,<EOL>aws_access_key_id=aws_id,<EOL>aws_secret_access_key=aws_secret,<EOL>surrogate_key=build_resource['<STR_LIT>'],<EOL>cache_control='<STR_LIT>',<EOL>surrogate_control=None,<EOL>upload_dir_redirect_objects=True)<EOL>logger.debug('<STR_LIT>', build_resource['<STR_LIT>'])<EOL>confirm_build(<EOL>build_resource['<STR_LIT>'],<EOL>ctx.obj['<STR_LIT>']<EOL>)<EOL>logger.debug('<STR_LIT>', build_resource['<STR_LIT>'])<EOL>

Upload a new site build to LSST the Docs.


def _should_skip_travis_event(on_travis_push, on_travis_pr, on_travis_api,<EOL>on_travis_cron):

travis_event = os.getenv('<STR_LIT>')<EOL>if travis_event is None:<EOL><INDENT>raise click.UsageError(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>')<EOL><DEDENT>if travis_event == '<STR_LIT>' and on_travis_push is False:<EOL><INDENT>click.echo('<STR_LIT>')<EOL>return True<EOL><DEDENT>elif travis_event == '<STR_LIT>' and on_travis_pr is False:<EOL><INDENT>click.echo('<STR_LIT>')<EOL>return True<EOL><DEDENT>elif travis_event == '<STR_LIT>' and on_travis_api is False:<EOL><INDENT>click.echo('<STR_LIT>')<EOL>return True<EOL><DEDENT>elif travis_event == '<STR_LIT>' and on_travis_cron is False:<EOL><INDENT>click.echo('<STR_LIT>')<EOL>return True<EOL><DEDENT>else:<EOL><INDENT>return False<EOL><DEDENT>

Detect if the upload should be skipped based on the
    ``TRAVIS_EVENT_TYPE`` environment variable.

    Returns
    -------
    should_skip : `bool`
        True if the upload should be skipped based on the combination of
        ``TRAVIS_EVENT_TYPE`` and user settings.


ch = logging.StreamHandler()<EOL>formatter = logging.Formatter(<EOL>'<STR_LIT>')<EOL>ch.setFormatter(formatter)<EOL>logger = logging.getLogger('<STR_LIT>')<EOL>logger.addHandler(ch)<EOL>logger.setLevel(log_level.upper())<EOL>ctx.obj = {<EOL>'<STR_LIT>': keeper_hostname,<EOL>'<STR_LIT:username>': username,<EOL>'<STR_LIT:password>': password,<EOL>'<STR_LIT>': None<EOL>}<EOL>

ltd is a command-line client for LSST the Docs.

    Use ltd to upload new site builds, and to work with the LTD Keeper API.


@main.command()<EOL>@click.argument('<STR_LIT>', default=None, required=False, nargs=<NUM_LIT:1>)<EOL>@click.pass_context<EOL>def help(ctx, topic, **kw):

<EOL>if topic is None:<EOL><INDENT>click.echo(ctx.parent.get_help())<EOL><DEDENT>else:<EOL><INDENT>click.echo(main.commands[topic].get_help(ctx))<EOL><DEDENT>


logger = logging.getLogger(__name__)<EOL>logger.info('<STR_LIT>', __name__)<EOL>if ctx.obj['<STR_LIT>'] is None:<EOL><INDENT>if ctx.obj['<STR_LIT:username>'] is None or ctx.obj['<STR_LIT:password>'] is None:<EOL><INDENT>raise click.UsageError(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>')<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>logger.debug(<EOL>'<STR_LIT>',<EOL>ctx.obj['<STR_LIT:username>'],<EOL>ctx.obj['<STR_LIT>'])<EOL>token = get_keeper_token(<EOL>ctx.obj['<STR_LIT>'],<EOL>ctx.obj['<STR_LIT:username>'],<EOL>ctx.obj['<STR_LIT:password>'])<EOL>ctx.obj['<STR_LIT>'] = token<EOL>logger.debug(<EOL>'<STR_LIT>',<EOL>ctx.obj['<STR_LIT:username>'],<EOL>ctx.obj['<STR_LIT>'])<EOL><DEDENT>else:<EOL><INDENT>logger.debug(<EOL>'<STR_LIT>')<EOL><DEDENT>

Ensure a token is in the Click context object or authenticate and obtain
    the token from LTD Keeper.

    Parameters
    ----------
    ctx : `click.Context`
        The Click context. ``ctx.obj`` must be a `dict` that contains keys:
        ``keeper_hostname``, ``username``, ``password``, ``token``. This
        context object is prepared by the main Click group,
        `ltdconveyor.cli.main.main`.


def register_build(host, keeper_token, product, git_refs):

data = {<EOL>'<STR_LIT>': git_refs<EOL>}<EOL>endpoint_url = uritemplate.expand(<EOL>urljoin(host, '<STR_LIT>'),<EOL>p=product)<EOL>r = requests.post(<EOL>endpoint_url,<EOL>auth=(keeper_token, '<STR_LIT>'),<EOL>json=data)<EOL>if r.status_code != <NUM_LIT>:<EOL><INDENT>raise KeeperError(r.json())<EOL><DEDENT>build_info = r.json()<EOL>return build_info<EOL>

Register a new build for a product on LSST the Docs.

    Wraps ``POST /products/{product}/builds/``.

    Parameters
    ----------
    host : `str`
        Hostname of LTD Keeper API server.
    keeper_token : `str`
        Auth token (`ltdconveyor.keeper.get_keeper_token`).
    product : `str`
        Name of the product in the LTD Keeper service.
    git_refs : `list` of `str`
        List of Git refs that correspond to the version of the build. Git refs
        can be tags or branches.

    Returns
    -------
    build_info : `dict`
        LTD Keeper build resource.

    Raises
    ------
    ltdconveyor.keeper.KeeperError
        Raised if there is an error communicating with the LTD Keeper API.


data = {<EOL>'<STR_LIT>': True<EOL>}<EOL>r = requests.patch(<EOL>build_url,<EOL>auth=(keeper_token, '<STR_LIT>'),<EOL>json=data)<EOL>if r.status_code != <NUM_LIT:200>:<EOL><INDENT>raise KeeperError(r)<EOL><DEDENT>

Confirm a build upload is complete.

    Wraps ``PATCH /builds/{build}``.

    Parameters
    ----------
    build_url : `str`
        URL of the build resource. Given a build resource, this URL is
        available from the ``self_url`` field.
    keeper_token : `str`
        Auth token (`ltdconveyor.keeper.get_keeper_token`).

    Raises
    ------
    ltdconveyor.keeper.KeeperError
        Raised if there is an error communicating with the LTD Keeper API.


def get_keeper_token(host, username, password):

token_endpoint = urljoin(host, '<STR_LIT>')<EOL>r = requests.get(token_endpoint, auth=(username, password))<EOL>if r.status_code != <NUM_LIT:200>:<EOL><INDENT>raise KeeperError('<STR_LIT>'.<EOL>format(host, r.status_code, r.json()))<EOL><DEDENT>return r.json()['<STR_LIT>']<EOL>

Get a temporary auth token from LTD Keeper.

    Parameters
    ----------
    host : `str`
        Hostname of the LTD Keeper API (e.g., ``'https://keeper.lsst.codes'``).
    username : `str`
        Username.
    password : `str`
        Password.

    Returns
    -------
    token : `str`
        LTD Keeper API token.

    Raises
    ------
    KeeperError
        Raised if the LTD Keeper API cannot return a token.


self.source = open(self.source, '<STR_LIT:rb>')    <EOL>self.loaded = True<EOL>


self.source.close()<EOL>self.loaded = False<EOL>

Closes the input file to free resources.


<EOL>reader = csv.reader(self.source)<EOL>headers = reader.next()<EOL>for row in reader:<EOL><INDENT>if not row:<EOL><INDENT>continue<EOL><DEDENT>yield dict(zip(headers, row))<EOL><DEDENT>

Iterator to read the rows of the CSV file.


val = item.get(source_name.encode('<STR_LIT:utf-8>'), None)<EOL>if val is not None:<EOL><INDENT>val = convert_string(val)<EOL><DEDENT>return val<EOL>

This method receives an item from the source and a source name,
and returns the text content for the `source_name` node.


def save_error(self, data, exception_info):

<EOL>self.errors.append({'<STR_LIT:data>': data,<EOL>'<STR_LIT>': '<STR_LIT>'.join(format_exception(*exception_info)),<EOL>})<EOL>


<EOL>if not self.loaded:<EOL><INDENT>self.load(self.source)<EOL><DEDENT>for item in self.get_items():<EOL><INDENT>data = self.parse_item(item)<EOL>instance = self.get_instance(data)<EOL>self.feed_instance(data, instance)<EOL>try:<EOL><INDENT>self.save_item(item, data, instance)<EOL><DEDENT>except Exception as e:<EOL><INDENT>self.save_error(data, sys.exc_info())<EOL><DEDENT><DEDENT>self.unload()<EOL>

Parses all data from the source, saving model instances.


<EOL>parsed_data = {}<EOL>for field_name in self.fields:<EOL><INDENT>source_name = self.field_map.get(field_name, field_name)<EOL>parse = getattr(self, '<STR_LIT>' % field_name, None)<EOL>if parse:<EOL><INDENT>value = parse(item, field_name, source_name)<EOL><DEDENT>else:<EOL><INDENT>value = self.get_value(item, source_name)<EOL><DEDENT>parsed_data[field_name] = value<EOL><DEDENT>return parsed_data<EOL>

Receives an item and returns a dictionary of field values.


<EOL>unique_fields = self.unique_fields<EOL>if not unique_fields:<EOL><INDENT>return self.model()<EOL><DEDENT>filter = dict([(f, data[f]) for f in unique_fields])<EOL>try:<EOL><INDENT>instance = self.model._default_manager.get(**filter)<EOL><DEDENT>except self.model.DoesNotExist:<EOL><INDENT>return self.model()<EOL><DEDENT>return instance<EOL>

Get an item from the database or an empty one if not found.


def feed_instance(self, data, instance):

for prop, val in data.items():<EOL><INDENT>setattr(instance, prop, val)<EOL><DEDENT>return instance<EOL>

Feeds a model instance using parsed data (usually from `parse_item`).


def save_item(self, item, data, instance, commit=True):

if commit:<EOL><INDENT>instance.save()<EOL><DEDENT>return instance<EOL>


self.loaded = False<EOL>

Unloads the source file.
Useful to close open files and free resources.

Not a required method by subclasses.

Rembember to unset the ``loaded`` when extending this method.


raise NotImplementedError<EOL>

Load data from the source.

Must be implemented on subclasses or will raise `NotImplementedError`.


raise NotImplementedError<EOL>

Get the list of items for the current source.
Must return an iterable.

Must be implemented on subclasses or will raise `NotImplementedError`.


raise NotImplementedError<EOL>

Receives an item from the source and a source name,
and must return the value for the field.

Must be implemented on subclasses or will raise `NotImplementedError`.


self.source = source<EOL>self.loaded = True<EOL>

Doesn't load anything, just changes the source property,
`ElementTree` is capable of using a file stream or path,
so the source type doesn't matter.


<EOL>for event, item in ElementTree.iterparse(self.source):<EOL><INDENT>if item.tag == self.item_tag_name:<EOL><INDENT>yield item<EOL>item.clear()<EOL><DEDENT><DEDENT>

Iterator of the list of items in the XML source.


return force_text(smart_str(item.findtext(source_name))).strip()<EOL>


<EOL>request = urllib2.Request(url)<EOL>request.add_header('<STR_LIT>', '<STR_LIT>')<EOL>opener = urllib2.build_opener()<EOL>response = opener.open(request)<EOL>data = response.read()<EOL>if response.headers.get('<STR_LIT>', '<STR_LIT>') == '<STR_LIT>':<EOL><INDENT>stream = StringIO.StringIO(data)<EOL>gzipper = gzip.GzipFile(fileobj=stream)<EOL>data = gzipper.read()<EOL><DEDENT>f = open(dest, '<STR_LIT:wb>')<EOL>f.write(data)<EOL>f.close()<EOL>

Downloads a HTTP resource from `url` and save to `dest`.

Capable of dealing with Gzip compressed content.


def parse_category(self, item, field_name, source_name):

<EOL>slug = category_map.get(self.get_value(item, source_name), None)<EOL>if not slug:<EOL><INDENT>return None<EOL><DEDENT>try:<EOL><INDENT>return Category.objects.get(slug=slug)<EOL><DEDENT>except Category.DoesNotExist:<EOL><INDENT>pass<EOL><DEDENT>

Converts the text category to a tasks.Category instance.


def parse_date(self, item, field_name, source_name):

<EOL>now = datetime.now().date()<EOL>val = self.get_value(item, source_name)<EOL>week_day, day = val.split()<EOL>day = int(day)<EOL>if now.day < day:<EOL><INDENT>if now.month == <NUM_LIT:1>:<EOL><INDENT>now = now.replace(month=<NUM_LIT:12>, year=now.year-<NUM_LIT:1>)<EOL><DEDENT>else:<EOL><INDENT>now = now.replace(month=now.month-<NUM_LIT:1>)<EOL><DEDENT><DEDENT>now = now.replace(day=day)<EOL>return now<EOL>

Converts the date in the format: Thu 03.

As only the day is provided, tries to find the best match
based on the current date, considering that dates are on
the past.


def parse_totals(self, item, field_name, source_name):

val = self.get_value(item, source_name)<EOL>try:<EOL><INDENT>return int(val)<EOL><DEDENT>except:<EOL><INDENT>return <NUM_LIT:0><EOL><DEDENT>


def run_population(population, evolution, gpus):

population_size = len(population)<EOL>for k in range(population_size // len(gpus) + <NUM_LIT:1>):<EOL><INDENT>procs = []<EOL>for j in range(len(gpus)):<EOL><INDENT>i = k * len(gpus) + j<EOL>if i < population_size:<EOL><INDENT>save_path = expand_path(<EOL>evolution.get_value_from_config(parse_config(population[i]),<EOL>evolution.path_to_models_save_path))<EOL>save_path.mkdir(parents=True, exist_ok=True)<EOL>f_name = save_path / "<STR_LIT>"<EOL>save_json(population[i], f_name)<EOL>with save_path.joinpath('<STR_LIT>').open('<STR_LIT:w>', encoding='<STR_LIT:utf8>') as outlog,save_path.joinpath('<STR_LIT>').open('<STR_LIT:w>', encoding='<STR_LIT:utf8>') as errlog:<EOL><INDENT>env = dict(os.environ)<EOL>if len(gpus) > <NUM_LIT:1> or gpus[<NUM_LIT:0>] != -<NUM_LIT:1>:<EOL><INDENT>env['<STR_LIT>'] = str(gpus[j])<EOL><DEDENT>procs.append(Popen("<STR_LIT>".format(sys.executable, str(f_name)),<EOL>shell=True, stdout=outlog, stderr=errlog, env=env))<EOL><DEDENT><DEDENT><DEDENT>for j, proc in enumerate(procs):<EOL><INDENT>i = k * len(gpus) + j<EOL>log.info(f'<STR_LIT>')<EOL>if proc.wait() != <NUM_LIT:0>:<EOL><INDENT>save_path = expand_path(<EOL>evolution.get_value_from_config(parse_config(population[i]),<EOL>evolution.path_to_models_save_path))<EOL>with save_path.joinpath('<STR_LIT>').open(encoding='<STR_LIT:utf8>') as errlog:<EOL><INDENT>log.warning(f'<STR_LIT>' +<EOL>errlog.read())<EOL><DEDENT><DEDENT><DEDENT><DEDENT>return None<EOL>

Change save and load paths for obtained population, save config.json with model config,
run population via current python executor (with which evolve.py already run)
and on given devices (-1 means CPU, other integeres - visible for evolve.py GPUs)
Args:
    population: list of dictionaries - configs of current population
    evolution: ParamsEvolution
    gpus: list of given devices (list of integers)

Returns:
    None


def __call__(self, utterances_batch: List[str], history_batch: List[List[str]],<EOL>states_batch: Optional[list] = None) -> Tuple[List[str], List[float]]:

responses, confidences = self.model(utterances_batch)<EOL>if isinstance(confidences[<NUM_LIT:0>], list):<EOL><INDENT>confidences = [max(c) for c in confidences]<EOL><DEDENT>return responses, confidences<EOL>

It returns the skill inference result.

        Output is batches of the skill inference results and estimated confidences.

        Args:
            utterances_batch: A batch of utterances.
            history_batch: A batch of list typed histories for each utterance.
            states_batch: Optional. A batch of arbitrary typed states for
                each utterance.

        Returns:
            Batches of the skill inference results and estimated confidences.


@staticmethod<EOL><INDENT>def build(data_path: str) -> Path:<DEDENT>

return Path(data_path)<EOL>

Base method that interprets ``data_path`` argument.

        Args:
            data_path: path to the tsv-file containing erroneous and corrected words

        Returns:
            the same path as a :class:`~pathlib.Path` object


@classmethod<EOL><INDENT>def read(cls, data_path: str, *args, **kwargs) -> Dict[str, List[Tuple[str, str]]]:<DEDENT>

fname = cls.build(data_path)<EOL>with fname.open(newline='<STR_LIT>', encoding='<STR_LIT:utf8>') as tsvfile:<EOL><INDENT>reader = csv.reader(tsvfile, delimiter='<STR_LIT:\t>')<EOL>next(reader)<EOL>res = [(mistake, correct) for mistake, correct in reader]<EOL><DEDENT>return {'<STR_LIT:train>': res}<EOL>

Read train data for spelling corrections algorithms

        Args:
            data_path: path that needs to be interpreted with :meth:`~deeppavlov.dataset_readers.typos_reader.TyposCustom.build`

        Returns:
            train data to pass to a :class:`~deeppavlov.dataset_iterators.typos_iterator.TyposDatasetIterator`


data_path = Path(data_path) / '<STR_LIT>'<EOL>fname = data_path / '<STR_LIT>'<EOL>if not is_done(data_path):<EOL><INDENT>url = '<STR_LIT>'<EOL>page = requests.get(url)<EOL>tree = html.fromstring(page.content)<EOL>raw = tree.xpath('<STR_LIT>')[<NUM_LIT:0>].splitlines()<EOL>data = []<EOL>for pair in raw:<EOL><INDENT>typo, corrects = pair.strip().split('<STR_LIT>')<EOL>for correct in corrects.split('<STR_LIT:U+002C>'):<EOL><INDENT>data.append([typo.strip(), correct.strip()])<EOL><DEDENT><DEDENT>fname.parent.mkdir(parents=True, exist_ok=True)<EOL>with fname.open('<STR_LIT:w>', newline='<STR_LIT>', encoding='<STR_LIT:utf8>') as tsvfile:<EOL><INDENT>writer = csv.writer(tsvfile, delimiter='<STR_LIT:\t>')<EOL>for line in data:<EOL><INDENT>writer.writerow(line)<EOL><DEDENT><DEDENT>mark_done(data_path)<EOL>log.info('<STR_LIT>')<EOL><DEDENT>return fname<EOL>

Download and parse common misspellings list from `Wikipedia <https://en.wikipedia.org/wiki/Wikipedia:Lists_of_common_misspellings/For_machines>`_

        Args:
            data_path: target directory to download the data to

        Returns:
            path to the resulting tsv-file


data_path = Path(data_path) / '<STR_LIT>'<EOL>fname = data_path / '<STR_LIT>'<EOL>if not is_done(data_path):<EOL><INDENT>url = '<STR_LIT>'<EOL>download(fname, url)<EOL>mark_done(data_path)<EOL>log.info('<STR_LIT>')<EOL><DEDENT>return fname<EOL>

Download misspellings list from `github <https://raw.githubusercontent.com/dkulagin/kartaslov/master/dataset/orfo_and_typos/orfo_and_typos.L1_5.csv>`_

        Args:
            data_path: target directory to download the data to

        Returns:
            path to the resulting csv-file


@staticmethod<EOL><INDENT>def read(data_path: str, *args, **kwargs) -> Dict[str, List[Tuple[str, str]]]:<DEDENT>

fname = TyposKartaslov.build(data_path)<EOL>with open(str(fname), newline='<STR_LIT>', encoding='<STR_LIT:utf8>') as csvfile:<EOL><INDENT>reader = csv.reader(csvfile, delimiter='<STR_LIT:;>')<EOL>next(reader)<EOL>res = [(mistake, correct) for correct, mistake, weight in reader]<EOL><DEDENT>return {'<STR_LIT:train>': res}<EOL>

Read train data for spelling corrections algorithms

        Args:
            data_path: path that needs to be interpreted with :meth:`~deeppavlov.dataset_readers.typos_reader.TyposKartaslov.build`

        Returns:
            train data to pass to a :class:`~deeppavlov.dataset_iterators.typos_iterator.TyposDatasetIterator`


def read(self,<EOL>data_path: str,<EOL>seed: int = None, *args, **kwargs) -> Dict[str, List[Tuple[List[str], int]]]:

data_path = expand_path(data_path)<EOL>train_fname = data_path / '<STR_LIT>'<EOL>test_fname = data_path / '<STR_LIT>'<EOL>train_data = self.build_data(train_fname)<EOL>test_data = self.build_data(test_fname)<EOL>dataset = {"<STR_LIT:train>": train_data, "<STR_LIT>": test_data, "<STR_LIT:test>": test_data}<EOL>return dataset<EOL>

Read the pretraining dataset for the paraphrase identification task from files.

        Args:
            data_path: A path to a folder with dataset files.
            seed: Random seed.


def read(self, data_path: str = None, *args, **kwargs) -> Dict:

with open(data_path) as f:<EOL><INDENT>content = f.readlines()<EOL><DEDENT>dataset = dict()<EOL>dataset["<STR_LIT:train>"] = [(line, ) for line in content]<EOL>dataset["<STR_LIT>"] = []<EOL>dataset["<STR_LIT:test>"] = []<EOL>return dataset<EOL>

Read lines from txt file

        Args:
            data_path: path to txt file

        Returns:
            A dictionary containing training, validation and test parts of the dataset obtainable via ``train``, ``valid`` and ``test`` keys.


@classmethod<EOL><INDENT>@overrides<EOL>def read(self, data_path: str, dialogs: bool = False) -> Dict[str, List]:<DEDENT>

required_files = (self._data_fname(dt) for dt in ('<STR_LIT:train>', '<STR_LIT>', '<STR_LIT:test>'))<EOL>if not all(Path(data_path, f).exists() for f in required_files):<EOL><INDENT>log.info('<STR_LIT>'.format(self.url, data_path))<EOL>download_decompress(self.url, data_path)<EOL>mark_done(data_path)<EOL><DEDENT>data = {<EOL>'<STR_LIT:train>': self._read_from_file(<EOL>Path(data_path, self._data_fname('<STR_LIT:train>')), dialogs),<EOL>'<STR_LIT>': self._read_from_file(<EOL>Path(data_path, self._data_fname('<STR_LIT>')), dialogs),<EOL>'<STR_LIT:test>': self._read_from_file(<EOL>Path(data_path, self._data_fname('<STR_LIT:test>')), dialogs)<EOL>}<EOL>return data<EOL>

Downloads ``'kvrest_public.tar.gz'``, decompresses, saves files to ``data_path``.

Parameters:
    data_path: path to save data
    dialogs: flag indices whether to output list of turns or list of dialogs

Returns:
    dictionary with ``'train'`` containing dialogs from ``'kvret_train_public.json'``, ``'valid'`` containing dialogs from ``'kvret_valid_public.json'``, ``'test'`` containing dialogs from ``'kvret_test_public.json'``. Each fields is a list of tuples ``(x_i, y_i)``.


@classmethod<EOL><INDENT>def _read_from_file(cls, file_path, dialogs=False):<DEDENT>

log.info("<STR_LIT>".format(file_path))<EOL>utterances, responses, dialog_indices =cls._get_turns(cls._iter_file(file_path), with_indices=True)<EOL>data = list(map(cls._format_turn, zip(utterances, responses)))<EOL>if dialogs:<EOL><INDENT>return [data[idx['<STR_LIT:start>']:idx['<STR_LIT:end>']] for idx in dialog_indices]<EOL><DEDENT>return data<EOL>


def read(self, data_path: str = None, data_url: str = None, x_col_name: str = '<STR_LIT:x>', y_col_name: str = '<STR_LIT:y>') -> Dict:

if data_url is not None:<EOL><INDENT>data = read_csv(data_url)<EOL><DEDENT>elif data_path is not None:<EOL><INDENT>data = read_csv(data_path)<EOL><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>x = data[x_col_name]<EOL>y = data[y_col_name]<EOL>train_xy_tuples = [(x[i].strip(), y[i].strip()) for i in range(len(x))]<EOL>dataset = dict()<EOL>dataset["<STR_LIT:train>"] = train_xy_tuples<EOL>dataset["<STR_LIT>"] = []<EOL>dataset["<STR_LIT:test>"] = []<EOL>return dataset<EOL>

Read FAQ dataset from specified csv file or remote url

Parameters:
    data_path: path to csv file of FAQ
    data_url: url to csv file of FAQ
    x_col_name: name of Question column in csv file
    y_col_name: name of Answer column in csv file

Returns:
    A dictionary containing training, validation and test parts of the dataset obtainable via
    ``train``, ``valid`` and ``test`` keys.


required_files = (self._data_fname(dt) for dt in ('<STR_LIT>', '<STR_LIT>', '<STR_LIT>'))<EOL>if not all(Path(data_path, f).exists() for f in required_files):<EOL><INDENT>log.info('<STR_LIT>'.format(self.url, data_path))<EOL>download_decompress(self.url, data_path)<EOL>mark_done(data_path)<EOL><DEDENT>data = {<EOL>'<STR_LIT:train>': self._read_from_file(<EOL>Path(data_path, self._data_fname('<STR_LIT>')), dialogs),<EOL>'<STR_LIT>': self._read_from_file(<EOL>Path(data_path, self._data_fname('<STR_LIT>')), dialogs),<EOL>'<STR_LIT:test>': self._read_from_file(<EOL>Path(data_path, self._data_fname('<STR_LIT>')), dialogs)<EOL>}<EOL>return data<EOL>

Downloads ``'dstc2_v2.tar.gz'`` archive from ipavlov internal server,
decompresses and saves files to ``data_path``.

Parameters:
    data_path: path to save DSTC2 dataset
    dialogs: flag which indicates whether to output list of turns or
     list of dialogs

Returns:
    dictionary that contains ``'train'`` field with dialogs from
    ``'dstc2-trn.jsonlist'``, ``'valid'`` field with dialogs from
    ``'dstc2-val.jsonlist'`` and ``'test'`` field with dialogs from
    ``'dstc2-tst.jsonlist'``. Each field is a list of tuples ``(x_i, y_i)``.


def read(self, data_path: str,<EOL>num_context_turns: int = <NUM_LIT:1>,<EOL>*args, **kwargs) -> Dict[str, List[Tuple[List[str], int]]]:

self.num_turns = num_context_turns<EOL>dataset = {'<STR_LIT:train>': None, '<STR_LIT>': None, '<STR_LIT:test>': None}<EOL>train_fname = Path(data_path) / '<STR_LIT>'<EOL>valid_fname = Path(data_path) / '<STR_LIT>'<EOL>test_fname = Path(data_path) / '<STR_LIT>'<EOL>self.sen2int_vocab = {}<EOL>self.classes_vocab_train = {}<EOL>self.classes_vocab_valid = {}<EOL>self.classes_vocab_test = {}<EOL>dataset["<STR_LIT:train>"] = self.preprocess_data_train(train_fname)<EOL>dataset["<STR_LIT>"] = self.preprocess_data_validation(valid_fname)<EOL>dataset["<STR_LIT:test>"] = self.preprocess_data_validation(test_fname)<EOL>return dataset<EOL>

Read the Ubuntu V2 dataset from csv files taking into account multi-turn dialogue ``context``.

        Args:
            data_path: A path to a folder with dataset csv files.
            num_context_turns: A maximum number of dialogue ``context`` turns.


@overrides<EOL><INDENT>def read(self, data_path: str, url: str = None,<EOL>format: str = "<STR_LIT>", class_sep: str = "<STR_LIT:U+002C>",<EOL>*args, **kwargs) -> dict:<DEDENT>

data_types = ["<STR_LIT:train>", "<STR_LIT>", "<STR_LIT:test>"]<EOL>train_file = kwargs.get('<STR_LIT:train>', '<STR_LIT>')<EOL>if not Path(data_path, train_file).exists():<EOL><INDENT>if url is None:<EOL><INDENT>raise Exception("<STR_LIT>".format(data_path))<EOL><DEDENT>log.info("<STR_LIT>".format(url, data_path))<EOL>download(source_url=url, dest_file_path=Path(data_path, train_file))<EOL><DEDENT>data = {"<STR_LIT:train>": [],<EOL>"<STR_LIT>": [],<EOL>"<STR_LIT:test>": []}<EOL>for data_type in data_types:<EOL><INDENT>file_name = kwargs.get(data_type, '<STR_LIT>'.format(data_type, format))<EOL>file = Path(data_path).joinpath(file_name)<EOL>if file.exists():<EOL><INDENT>if format == '<STR_LIT>':<EOL><INDENT>keys = ('<STR_LIT>', '<STR_LIT>', '<STR_LIT>')<EOL>options = {k: kwargs[k] for k in keys if k in kwargs}<EOL>df = pd.read_csv(file, **options)<EOL><DEDENT>elif format == '<STR_LIT>':<EOL><INDENT>keys = ('<STR_LIT>', '<STR_LIT>')<EOL>options = {k: kwargs[k] for k in keys if k in kwargs}<EOL>df = pd.read_json(file, **options)<EOL><DEDENT>else:<EOL><INDENT>raise Exception('<STR_LIT>'.format(format))<EOL><DEDENT>x = kwargs.get("<STR_LIT:x>", "<STR_LIT:text>")<EOL>y = kwargs.get('<STR_LIT:y>', '<STR_LIT>')<EOL>if isinstance(x, list):<EOL><INDENT>data[data_type] = [([row[x_] for x_ in x], str(row[y]).split(class_sep)) for _, row in df.iterrows()]<EOL><DEDENT>else:<EOL><INDENT>data[data_type] = [(row[x], str(row[y]).split(class_sep)) for _, row in df.iterrows()]<EOL><DEDENT><DEDENT>else:<EOL><INDENT>log.warning("<STR_LIT>".format(file))<EOL><DEDENT><DEDENT>return data<EOL>

Read dataset from data_path directory.
Reading files are all data_types + extension
(i.e for data_types=["train", "valid"] files "train.csv" and "valid.csv" form
data_path will be read)

Args:
    data_path: directory with files
    url: download data files if data_path not exists or empty
    format: extension of files. Set of Values: ``"csv", "json"``
    class_sep: string separator of labels in column with labels
    sep (str): delimeter for ``"csv"`` files. Default: ``","``
    header (int): row number to use as the column names
    names (array): list of column names to use
    orient (str): indication of expected JSON string format
    lines (boolean): read the file as a json object per line. Default: ``False``

Returns:
    dictionary with types from data_types.
    Each field of dictionary is a list of tuples (x_i, y_i)


def read(self, data_path: str, catalog: list, **kwargs) -> Dict[str, List[Tuple[Any, Any]]]:

logger.info(f"<STR_LIT>")<EOL>if not isinstance(catalog, list):<EOL><INDENT>catalog = [catalog]<EOL><DEDENT>ec_data_global: List[Any] = []<EOL>data_path = Path(expand_path(data_path))<EOL>if not is_done(data_path):<EOL><INDENT>self._download_data(data_path)<EOL><DEDENT>if data_path.is_dir():<EOL><INDENT>for fname in data_path.rglob("<STR_LIT>"):<EOL><INDENT>if any(cat in fname.name for cat in catalog):<EOL><INDENT>logger.info(f"<STR_LIT>")<EOL>ec_data_global += self._load_amazon_ecommerce_file(fname)<EOL><DEDENT><DEDENT><DEDENT>dataset = {<EOL>'<STR_LIT:train>': [((item['<STR_LIT>'], [], {}), item) for item in ec_data_global],<EOL>'<STR_LIT>': [],<EOL>'<STR_LIT:test>':  []<EOL>}<EOL>logger.info(f"<STR_LIT>")<EOL>return dataset<EOL>

Load data from specific catalog

        Parameters:
            data_path: where the dataset is located
            catalog: names of the specific subcategories

        Returns:
            dataset: loaded dataset


def _download_data(self, data_path: str) -> None:

url = "<STR_LIT>"<EOL>download_decompress(url, data_path)<EOL>mark_done(data_path)<EOL>


def _load_amazon_ecommerce_file(self, fname: str) -> List[Dict[Any, Any]]:

ec_data = []<EOL>item: Dict = {}<EOL>new_item_re = re.compile("<STR_LIT>")<EOL>with open(fname, '<STR_LIT:r>', encoding='<STR_LIT:utf-8>', errors='<STR_LIT:ignore>') as file:<EOL><INDENT>for line in file:<EOL><INDENT>if new_item_re.match(line):<EOL><INDENT>if len(item.keys()) > <NUM_LIT:0>:<EOL><INDENT>if '<STR_LIT>' in item and '<STR_LIT>' in item:<EOL><INDENT>ec_data.append(item)<EOL><DEDENT><DEDENT>item = {'<STR_LIT>': int(line[<NUM_LIT:5>:]), '<STR_LIT>': fname.name.split("<STR_LIT:_>")[<NUM_LIT:1>]}<EOL><DEDENT>else:<EOL><INDENT>row = line.strip().split("<STR_LIT:=>")<EOL>if len(row) == <NUM_LIT:2>:<EOL><INDENT>if row[<NUM_LIT:0>] in item:<EOL><INDENT>item[row[<NUM_LIT:0>]] += "<STR_LIT:.>" + row[<NUM_LIT:1>]<EOL><DEDENT>else:<EOL><INDENT>item[row[<NUM_LIT:0>]] = row[<NUM_LIT:1>]<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>return ec_data<EOL>

Parse dataset

        Parameters:
            fname: catalog file

        Returns:
            ec_data: parsed catalog data


return filepath.split("<STR_LIT:->")[<NUM_LIT:0>]<EOL>

Extracts language from typical UD filename


def read_infile(infile: Union[Path, str], from_words=False,<EOL>word_column: int = WORD_COLUMN, pos_column: int = POS_COLUMN,<EOL>tag_column: int = TAG_COLUMN, max_sents: int = -<NUM_LIT:1>,<EOL>read_only_words: bool = False) -> List[Tuple[List, Union[List, None]]]:

answer, curr_word_sent, curr_tag_sent = [], [], []<EOL>if from_words:<EOL><INDENT>word_column, read_only_words = <NUM_LIT:0>, True<EOL><DEDENT>with open(infile, "<STR_LIT:r>", encoding="<STR_LIT:utf8>") as fin:<EOL><INDENT>for line in fin:<EOL><INDENT>line = line.strip()<EOL>if line.startswith("<STR_LIT:#>"):<EOL><INDENT>continue<EOL><DEDENT>if line == "<STR_LIT>":<EOL><INDENT>if len(curr_word_sent) > <NUM_LIT:0>:<EOL><INDENT>if read_only_words:<EOL><INDENT>curr_tag_sent = None<EOL><DEDENT>answer.append((curr_word_sent, curr_tag_sent))<EOL><DEDENT>curr_tag_sent, curr_word_sent = [], []<EOL>if len(answer) == max_sents:<EOL><INDENT>break<EOL><DEDENT>continue<EOL><DEDENT>splitted = line.split("<STR_LIT:\t>")<EOL>index = splitted[<NUM_LIT:0>]<EOL>if not from_words and not index.isdigit():<EOL><INDENT>continue<EOL><DEDENT>curr_word_sent.append(splitted[word_column])<EOL>if not read_only_words:<EOL><INDENT>pos, tag = splitted[pos_column], splitted[tag_column]<EOL>tag = pos if tag == "<STR_LIT:_>" else "<STR_LIT>".format(pos, tag)<EOL>curr_tag_sent.append(tag)<EOL><DEDENT><DEDENT>if len(curr_word_sent) > <NUM_LIT:0>:<EOL><INDENT>if read_only_words:<EOL><INDENT>curr_tag_sent = None<EOL><DEDENT>answer.append((curr_word_sent, curr_tag_sent))<EOL><DEDENT><DEDENT>return answer<EOL>

Reads input file in CONLL-U format

    Args:
        infile: a path to a file
        word_column: column containing words (default=1)
        pos_column: column containing part-of-speech labels (default=3)
        tag_column: column containing fine-grained tags (default=5)
        max_sents: maximal number of sents to read
        read_only_words: whether to read only words

    Returns:
        a list of sentences. Each item contains a word sequence and a tag sequence, which is ``None``
        in case ``read_only_words = True``


def read(self, data_path: Union[List, str],<EOL>language: Optional[None] = None,<EOL>data_types: Optional[List[str]] = None,<EOL>**kwargs) -> Dict[str, List]:

if data_types is None:<EOL><INDENT>data_types = ["<STR_LIT:train>", "<STR_LIT>"]<EOL><DEDENT>elif isinstance(data_types, str):<EOL><INDENT>data_types = list(data_types)<EOL><DEDENT>for data_type in data_types:<EOL><INDENT>if data_type not in ["<STR_LIT:train>", "<STR_LIT>", "<STR_LIT:test>"]:<EOL><INDENT>raise ValueError("<STR_LIT>"<EOL>"<STR_LIT>".format(data_type))<EOL><DEDENT><DEDENT>if isinstance(data_path, str):<EOL><INDENT>data_path = Path(data_path)<EOL><DEDENT>if isinstance(data_path, Path):<EOL><INDENT>if data_path.exists():<EOL><INDENT>is_file = data_path.is_file()<EOL><DEDENT>else:<EOL><INDENT>is_file = (len(data_types) == <NUM_LIT:1>)<EOL><DEDENT>if is_file:<EOL><INDENT>data_path, reserve_data_path = [data_path], None<EOL><DEDENT>else:<EOL><INDENT>if language is None:<EOL><INDENT>raise ValueError("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>reserve_data_path = data_path<EOL>data_path = [data_path / "<STR_LIT>".format(language, mode)<EOL>for mode in data_types]<EOL>reserve_data_path = [<EOL>reserve_data_path / language / "<STR_LIT>".format(language, mode)<EOL>for mode in data_types]<EOL><DEDENT><DEDENT>else:<EOL><INDENT>data_path = [Path(data_path) for data_path in data_path]<EOL>reserve_data_path = None<EOL><DEDENT>if len(data_path) != len(data_types):<EOL><INDENT>raise ValueError("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>has_missing_files = any(not filepath.exists() for filepath in data_path)<EOL>if has_missing_files and reserve_data_path is not None:<EOL><INDENT>has_missing_files = any(not filepath.exists() for filepath in reserve_data_path)<EOL>if not has_missing_files:<EOL><INDENT>data_path = reserve_data_path<EOL><DEDENT><DEDENT>if has_missing_files:<EOL><INDENT>dir_path = data_path[<NUM_LIT:0>].parent<EOL>language = language or get_language(data_path[<NUM_LIT:0>].parts[-<NUM_LIT:1>])<EOL>url = self.URL + "<STR_LIT>".format(language)<EOL>log.info('<STR_LIT>'.format(url, dir_path))<EOL>dir_path.mkdir(exist_ok=True, parents=True)<EOL>download_decompress(url, dir_path)<EOL>mark_done(dir_path)<EOL><DEDENT>data = {}<EOL>for mode, filepath in zip(data_types, data_path):<EOL><INDENT>if mode == "<STR_LIT>":<EOL><INDENT>mode = "<STR_LIT>"<EOL>

Reads UD dataset from data_path.

        Args:
            data_path: can be either
                1. a directory containing files. The file for data_type 'mode'
                is then data_path / {language}-ud-{mode}.conllu
                2. a list of files, containing the same number of items as data_types
            language: a language to detect filename when it is not given
            data_types: which dataset parts among 'train', 'dev', 'test' are returned

        Returns:
            a dictionary containing dataset fragments (see ``read_infile``) for given data types


def read(self,<EOL>data_path: str,<EOL>do_lower_case: bool = True,<EOL>seed: int = None, *args, **kwargs) -> Dict[str, List[Tuple[List[str], int]]]:

data_path = expand_path(data_path)<EOL>train_fname = data_path / '<STR_LIT>'<EOL>test_fname =  data_path / '<STR_LIT>'<EOL>train_data = self.build_data(train_fname, do_lower_case)<EOL>test_data = self.build_data(test_fname, do_lower_case)<EOL>dataset = {"<STR_LIT:train>": train_data, "<STR_LIT>": [], "<STR_LIT:test>": test_data}<EOL>return dataset<EOL>

Read the paraphraser.ru dataset from files.

        Args:
            data_path: A path to a folder with dataset files.
            do_lower_case: Do you want to lowercase all texts
            seed: Random seed.


def read(self, dir_path: str, dataset: Optional[str] = '<STR_LIT>', url: Optional[str] = None, *args, **kwargs)-> Dict[str, Dict[str, Any]]:

if url is not None:<EOL><INDENT>self.url = url<EOL><DEDENT>elif dataset == '<STR_LIT>':<EOL><INDENT>self.url = self.url_squad<EOL><DEDENT>elif dataset == '<STR_LIT>':<EOL><INDENT>self.url = self.url_sber_squad<EOL><DEDENT>elif dataset == '<STR_LIT>':<EOL><INDENT>self.url = self.url_multi_squad<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError('<STR_LIT>'.format(dataset))<EOL><DEDENT>dir_path = Path(dir_path)<EOL>required_files = ['<STR_LIT>'.format(dt) for dt in ['<STR_LIT:train>', '<STR_LIT>']]<EOL>if not dir_path.exists():<EOL><INDENT>dir_path.mkdir()<EOL><DEDENT>if not all((dir_path / f).exists() for f in required_files):<EOL><INDENT>download_decompress(self.url, dir_path)<EOL><DEDENT>dataset = {}<EOL>for f in required_files:<EOL><INDENT>with dir_path.joinpath(f).open('<STR_LIT:r>', encoding='<STR_LIT:utf8>') as fp:<EOL><INDENT>data = json.load(fp)<EOL><DEDENT>if f == '<STR_LIT>':<EOL><INDENT>dataset['<STR_LIT>'] = data<EOL><DEDENT>else:<EOL><INDENT>dataset['<STR_LIT:train>'] = data<EOL><DEDENT><DEDENT>return dataset<EOL>

Args:
    dir_path: path to save data
    dataset: default dataset names: ``'SQuAD'``, ``'SberSQuAD'`` or ``'MultiSQuAD'``
    url: link to archive with dataset, use url argument if non-default dataset is used

Returns:
    dataset split on train/valid

Raises:
    RuntimeError: if `dataset` is not one of these: ``'SQuAD'``, ``'SberSQuAD'``, ``'MultiSQuAD'``.


def read(self, dir_path: str, dataset: Optional[str] = '<STR_LIT>', url: Optional[str] = None, *args,<EOL>**kwargs) -> Dict[str, Dict[str, Any]]:

if url is not None:<EOL><INDENT>self.url = url<EOL><DEDENT>elif dataset == '<STR_LIT>':<EOL><INDENT>self.url = self.url_multi_squad_retr<EOL><DEDENT>elif dataset == '<STR_LIT>':<EOL><INDENT>self.url = self.url_multi_squad_ru_retr<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError('<STR_LIT>'.format(dataset))<EOL><DEDENT>dir_path = Path(dir_path)<EOL>required_files = ['<STR_LIT>'.format(dt) for dt in ['<STR_LIT:train>', '<STR_LIT>']]<EOL>if not dir_path.exists():<EOL><INDENT>dir_path.mkdir(parents=True)<EOL><DEDENT>if not all((dir_path / f).exists() for f in required_files):<EOL><INDENT>download_decompress(self.url, dir_path)<EOL><DEDENT>dataset = {}<EOL>for f in required_files:<EOL><INDENT>if '<STR_LIT>' in f:<EOL><INDENT>dataset['<STR_LIT>'] = dir_path.joinpath(f)<EOL><DEDENT>else:<EOL><INDENT>dataset['<STR_LIT:train>'] = dir_path.joinpath(f)<EOL><DEDENT><DEDENT>return dataset<EOL>

Args:
    dir_path: path to save data
    dataset: default dataset names: ``'MultiSQuADRetr'``, ``'MultiSQuADRuRetr'``
    url: link to archive with dataset, use url argument if non-default dataset is used

Returns:
    dataset split on train/valid

Raises:
    RuntimeError: if `dataset` is not one of these: ``'MultiSQuADRetr'``, ``'MultiSQuADRuRetr'``.


def read(self, data_path: Union[Path, str], db_url: Optional[str] = None, *args,<EOL>**kwargs) -> None:

logger.info('<STR_LIT>')<EOL>try:<EOL><INDENT>save_path = expand_path(kwargs['<STR_LIT>'])<EOL><DEDENT>except KeyError:<EOL><INDENT>raise ConfigError(<EOL>f'<STR_LIT>')<EOL><DEDENT>if save_path.exists() and save_path.with_suffix(f'<STR_LIT>').exists():<EOL><INDENT>return<EOL><DEDENT>try:<EOL><INDENT>dataset_format = kwargs['<STR_LIT>']<EOL><DEDENT>except KeyError:<EOL><INDENT>raise ConfigError(<EOL>f'<STR_LIT>')<EOL><DEDENT>save_path.parent.mkdir(parents=True, exist_ok=True)<EOL>if db_url:<EOL><INDENT>download_dir = save_path.parent<EOL>logger.info(f'<STR_LIT>')<EOL>download(download_dir, db_url, force_download=False)<EOL>return<EOL><DEDENT>self._build_db(save_path, dataset_format, expand_path(data_path))<EOL>

Build a SQLite database from provided files, download SQLite database from a provided URL,
         or do nothing.

        Args:
            data_path: a directory/file with texts to create a database from
            db_url: path to a database url
            kwargs:
                save_path: a path where a database should be saved to, or path to a ready database
                dataset_format: initial data format; should be selected from ['txt', 'wiki', 'json']

        Returns:
            None


def iter_files(self, path: Union[Path, str]) -> Generator[Path, Any, Any]:

path = Path(path)<EOL>if path.is_file():<EOL><INDENT>yield path<EOL><DEDENT>elif path.is_dir():<EOL><INDENT>for item in path.iterdir():<EOL><INDENT>yield from self.iter_files(item)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise RuntimeError("<STR_LIT>".format(path))<EOL><DEDENT>

Iterate over folder with files or a single file and generate file paths.

        Args:
            path: path to a folder or a file

        Raises:
            RuntimeError if the provided `path` doesn't exist

        Yields:
            file paths one by one

        Returns:
            None


def _build_db(self, save_path: Union[Path, str], dataset_format: str,<EOL>data_path: Union[Path, str],<EOL>num_workers: int = <NUM_LIT:8>) -> None:

done_path = save_path.with_suffix(f'<STR_LIT>')<EOL>if Path(save_path).exists():<EOL><INDENT>Path(save_path).unlink()<EOL><DEDENT>if done_path.exists():<EOL><INDENT>done_path.unlink()<EOL><DEDENT>logger.info('<STR_LIT>')<EOL>try:<EOL><INDENT>conn = sqlite3.connect(str(save_path))<EOL><DEDENT>except sqlite3.OperationalError as e:<EOL><INDENT>e.args = e.args + ("<STR_LIT>",)<EOL>raise e<EOL><DEDENT>c = conn.cursor()<EOL>sql_table = "<STR_LIT>"<EOL>c.execute(sql_table)<EOL>files = [f for f in self.iter_files(data_path)]<EOL>workers = Pool(num_workers)<EOL>if dataset_format == '<STR_LIT>':<EOL><INDENT>fn = self._get_file_contents<EOL><DEDENT>elif dataset_format == '<STR_LIT>':<EOL><INDENT>fn = self._get_json_contents<EOL><DEDENT>elif dataset_format == '<STR_LIT>':<EOL><INDENT>fn = self._get_wiki_contents<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError('<STR_LIT>')<EOL><DEDENT>with tqdm(total=len(files)) as pbar:<EOL><INDENT>for data in tqdm(workers.imap_unordered(fn, files)):<EOL><INDENT>try:<EOL><INDENT>c.executemany("<STR_LIT>", data)<EOL>pbar.update()<EOL><DEDENT>except sqlite3.IntegrityError as e:<EOL><INDENT>logger.warning(e)<EOL><DEDENT><DEDENT><DEDENT>conn.commit()<EOL>conn.close()<EOL>done_path.touch()<EOL>

Build a SQLite database in parallel and save it to a pointed path.

        Args:
            save_path: a path where the ready database should be saved
            dataset_format: a data format, should be selected from ['txt', 'json', 'wiki']
            data_path: path to a folder/file from which to build a database
            num_workers: a number of workers for parallel database building

        Raises:
            sqlite3.OperationalError if `save_path` doesn't exist.
            RuntimeError if dataset_format is not in ['txt', 'json', 'wiki']

        Returns:
            None


@staticmethod<EOL><INDENT>def _get_file_contents(fpath: Union[Path, str]) -> List[Tuple[str, str]]:<DEDENT>

with open(fpath, encoding='<STR_LIT:utf-8>') as fin:<EOL><INDENT>text = fin.read()<EOL>normalized_text = unicodedata.normalize('<STR_LIT>', text)<EOL>return [(fpath.name, normalized_text)]<EOL><DEDENT>

Extract file contents from '.txt' file.

        Args:
            fpath: path to a '.txt' file.

        Returns:
             a list with tuple of normalized file name and file contents


@staticmethod<EOL><INDENT>def _get_json_contents(fpath: Union[Path, str]) -> List[Tuple[str, str]]:<DEDENT>

docs = []<EOL>with open(fpath, encoding='<STR_LIT:utf-8>') as fin:<EOL><INDENT>for line in fin:<EOL><INDENT>data = json.loads(line)<EOL>for doc in data:<EOL><INDENT>if not doc:<EOL><INDENT>continue<EOL><DEDENT>text = doc['<STR_LIT:text>']<EOL>normalized_text = unicodedata.normalize('<STR_LIT>', text)<EOL>docs.append((doc['<STR_LIT:title>'], normalized_text))<EOL><DEDENT><DEDENT><DEDENT>return docs<EOL>

Extract file contents from '.json' file. JSON files should be formatted as list with dicts
        which contain 'title' and 'doc' keywords.

        Args:
            fpath: path to a '.json' file.

        Returns:
            a list with tuples of normalized file name and file contents


@staticmethod<EOL><INDENT>def _get_wiki_contents(fpath: Union[Path, str]) -> List[Tuple[str, str]]:<DEDENT>

docs = []<EOL>with open(fpath, encoding='<STR_LIT:utf-8>') as fin:<EOL><INDENT>for line in fin:<EOL><INDENT>doc = json.loads(line)<EOL>if not doc:<EOL><INDENT>continue<EOL><DEDENT>text = doc['<STR_LIT:text>']<EOL>normalized_text = unicodedata.normalize('<STR_LIT>', text)<EOL>docs.append((doc['<STR_LIT:title>'], normalized_text))<EOL><DEDENT><DEDENT>return docs<EOL>

Extract file contents from wiki extractor formatted files.

        Args:
            fpath: path to a '.txt' file in wiki extractor format

        Returns:
            a list with tuples of normalized file name and file contents


def read(self, data_path: str, **kwargs) -> Dict[str, List[Tuple[List[str], int]]]:

dataset = {'<STR_LIT:train>': None, '<STR_LIT>': None, '<STR_LIT:test>': None}<EOL>data_path = expand_path(data_path)<EOL>train_fname = data_path / '<STR_LIT>'<EOL>valid_fname = data_path / '<STR_LIT>'<EOL>test_fname = data_path / '<STR_LIT>'<EOL>dataset["<STR_LIT:train>"] = self._preprocess_data_train(train_fname)<EOL>dataset["<STR_LIT>"] = self._preprocess_data_valid_test(valid_fname)<EOL>dataset["<STR_LIT:test>"] = self._preprocess_data_valid_test(test_fname)<EOL>return dataset<EOL>

Read the dataset for ranking or paraphrase identification with Siamese networks.

        Args:
            data_path: A path to a folder with dataset files.


def read(self, data_path: str,<EOL>*args, **kwargs) -> Dict[str, List[Tuple[List[str], int]]]:

data_path = expand_path(data_path)<EOL>dataset = {'<STR_LIT:train>': None, '<STR_LIT>': None, '<STR_LIT:test>': None}<EOL>train_fname = Path(data_path) / '<STR_LIT>'<EOL>valid_fname = Path(data_path) / '<STR_LIT>'<EOL>test_fname = Path(data_path) / '<STR_LIT>'<EOL>self.sen2int_vocab = {}<EOL>self.classes_vocab_train = {}<EOL>self.classes_vocab_valid = {}<EOL>self.classes_vocab_test = {}<EOL>dataset["<STR_LIT:train>"] = self.preprocess_data_train(train_fname)<EOL>dataset["<STR_LIT>"] = self.preprocess_data_validation(valid_fname)<EOL>dataset["<STR_LIT:test>"] = self.preprocess_data_validation(test_fname)<EOL>return dataset<EOL>

Read the Ubuntu V2 dataset from csv files.

        Args:
            data_path: A path to a folder with dataset csv files.


data_path = expand_path(data_path)<EOL>dataset = {'<STR_LIT:train>': None, '<STR_LIT>': None, '<STR_LIT:test>': None}<EOL>train_fname = data_path / '<STR_LIT>'<EOL>valid_fname = data_path / '<STR_LIT>'<EOL>test_fname = data_path / '<STR_LIT>'<EOL>int2tok_fname = data_path / '<STR_LIT>'<EOL>response2ints_fname = data_path / '<STR_LIT>'<EOL>self.int2tok_vocab = self._build_int2tok_vocab(int2tok_fname)<EOL>self.idxs2cont_vocab = self._build_context2toks_vocab(train_fname, valid_fname, test_fname)<EOL>self.response2str_vocab = self._build_response2str_vocab(response2ints_fname)<EOL>dataset["<STR_LIT>"] = self._preprocess_data_valid_test(valid_fname)<EOL>dataset["<STR_LIT:train>"] = self._preprocess_data_train(train_fname)<EOL>dataset["<STR_LIT:test>"] = self._preprocess_data_valid_test(test_fname)<EOL>return dataset<EOL>

Read the InsuranceQA V1 dataset from files.

        Args:
            data_path: A path to a folder with dataset files.


try:<EOL><INDENT>predictions = [np.round(x) for x in y_predicted]<EOL><DEDENT>except TypeError:<EOL><INDENT>predictions = y_predicted<EOL><DEDENT>return f1_score(y_true, predictions)<EOL>

Calculates F1 (binary) measure.

Args:
    y_true: list of true values
    y_predicted: list of predicted values

Returns:
    F1 score


try:<EOL><INDENT>predictions = [np.round(x) for x in y_predicted]<EOL><DEDENT>except TypeError:<EOL><INDENT>predictions = y_predicted<EOL><DEDENT>return f1_score(np.array(y_true), np.array(predictions), average="<STR_LIT>")<EOL>

Calculates F1 macro measure.

Args:
    y_true: list of true values
    y_predicted: list of predicted values

Returns:
    F1 score


Calculates F1 weighted measure.

Args:
    y_true: list of true values
    y_predicted: list of predicted values

Returns:
    F1 score


@register_metric('<STR_LIT>')<EOL>def elmo_loss2ppl(losses: List[np.ndarray]) -> float:

avg_loss = np.mean(losses)<EOL>return float(np.exp(avg_loss))<EOL>

Calculates perplexity by loss

    Args:
        losses: list of numpy arrays of model losses

    Returns:
        perplexity : float


@register_metric('<STR_LIT>')<EOL>def sk_log_loss(y_true: Union[List[List[float]], List[List[int]], np.ndarray],<EOL>y_predicted: Union[List[List[float]], List[List[int]], np.ndarray]) -> float:

return log_loss(y_true, y_predicted)<EOL>

Calculates log loss.

Args:
    y_true: list or array of true values
    y_predicted: list or array of predicted values

Returns:
    Log loss


@register_metric('<STR_LIT>')<EOL>def accuracy(y_true: [list, np.ndarray], y_predicted: [list, np.ndarray]) -> float:

examples_len = len(y_true)<EOL>correct = sum([y1 == y2 for y1, y2 in zip(y_true, y_predicted)])<EOL>return correct / examples_len if examples_len else <NUM_LIT:0><EOL>

Calculate accuracy in terms of absolute coincidence

Args:
    y_true: array of true values
    y_predicted: array of predicted values

Returns:
    portion of absolutely coincidental samples


@register_metric('<STR_LIT>')<EOL>def sets_accuracy(y_true: [list, np.ndarray], y_predicted: [list, np.ndarray]) -> float:

examples_len = len(y_true)<EOL>correct = sum([set(y1) == set(y2) for y1, y2 in zip(y_true, y_predicted)])<EOL>return correct / examples_len if examples_len else <NUM_LIT:0><EOL>

Calculate accuracy in terms of sets coincidence

Args:
    y_true: true values
    y_predicted: predicted values

Returns:
    portion of samples with absolutely coincidental sets of predicted values


predictions = [round(x) for x in y_predicted]<EOL>examples_len = len(y_true)<EOL>correct = sum([y1 == y2 for y1, y2 in zip(y_true, predictions)])<EOL>return correct / examples_len if examples_len else <NUM_LIT:0><EOL>

Rounds predictions and calculates accuracy in terms of absolute coincidence.

Args:
    y_true: list of true values
    y_predicted: list of predicted values

Returns:
    portion of absolutely coincidental samples


def recall_at_k(y_true: List[int], y_pred: List[List[np.ndarray]], k: int):

num_examples = float(len(y_pred))<EOL>predictions = np.array(y_pred)<EOL>predictions = np.flip(np.argsort(predictions, -<NUM_LIT:1>), -<NUM_LIT:1>)[:, :k]<EOL>num_correct = <NUM_LIT:0><EOL>for el in predictions:<EOL><INDENT>if <NUM_LIT:0> in el:<EOL><INDENT>num_correct += <NUM_LIT:1><EOL><DEDENT><DEDENT>return float(num_correct) / num_examples<EOL>

Calculates recall at k ranking metric.

Args:
    y_true: Labels. Not used in the calculation of the metric.
    y_predicted: Predictions.
        Each prediction contains ranking score of all ranking candidates for the particular data sample.
        It is supposed that the ranking score for the true candidate goes first in the prediction.

Returns:
    Recall at k


@register_metric('<STR_LIT>')<EOL>def roc_auc_score(y_true: Union[List[List[float]], List[List[int]], np.ndarray],<EOL>y_pred: Union[List[List[float]], List[List[int]], np.ndarray]) -> float:

try:<EOL><INDENT>return sklearn.metrics.roc_auc_score(np.squeeze(np.array(y_true)),<EOL>np.squeeze(np.array(y_pred)), average="<STR_LIT>")<EOL><DEDENT>except ValueError:<EOL><INDENT>return <NUM_LIT:0.><EOL><DEDENT>

Compute Area Under the Curve (AUC) from prediction scores.

Args:
    y_true: true binary labels
    y_pred: target scores, can either be probability estimates of the positive class

Returns:
    Area Under the Curve (AUC) from prediction scores


@register_metric('<STR_LIT>')<EOL>def bleu_advanced(y_true: List[Any], y_predicted: List[Any],<EOL>weights: Tuple=(<NUM_LIT:1>,), smoothing_function=SMOOTH.method1,<EOL>auto_reweigh=False, penalty=True) -> float:

bleu_measure = sentence_bleu([y_true], y_predicted, weights, smoothing_function, auto_reweigh)<EOL>hyp_len = len(y_predicted)<EOL>hyp_lengths = hyp_len<EOL>ref_lengths = closest_ref_length([y_true], hyp_len)<EOL>bpenalty = brevity_penalty(ref_lengths, hyp_lengths)<EOL>if penalty is True or bpenalty == <NUM_LIT:0>:<EOL><INDENT>return bleu_measure<EOL><DEDENT>return bleu_measure/bpenalty<EOL>

Calculate BLEU score

    Parameters:
        y_true: list of reference tokens
        y_predicted: list of query tokens
        weights: n-gram weights
        smoothing_function: SmoothingFunction
        auto_reweigh: Option to re-normalize the weights uniformly
        penalty: either enable brevity penalty or not

    Return:
        BLEU score


ngram_counts = collections.Counter()<EOL>for order in range(<NUM_LIT:1>, max_order + <NUM_LIT:1>):<EOL><INDENT>for i in range(<NUM_LIT:0>, len(segment) - order + <NUM_LIT:1>):<EOL><INDENT>ngram = tuple(segment[i:i+order])<EOL>ngram_counts[ngram] += <NUM_LIT:1><EOL><DEDENT><DEDENT>return ngram_counts<EOL>

Extracts all n-grams upto a given maximum order from an input segment.

    Args:
      segment: text segment from which n-grams will be extracted.
      max_order: maximum length in tokens of the n-grams returned by this
          methods.

    Returns:
      The Counter containing all n-grams upto max_order in segment
      with a count of how many times each n-gram occurred.


def compute_bleu(reference_corpus, translation_corpus, max_order=<NUM_LIT:4>,<EOL>smooth=False):

matches_by_order = [<NUM_LIT:0>] * max_order<EOL>possible_matches_by_order = [<NUM_LIT:0>] * max_order<EOL>reference_length = <NUM_LIT:0><EOL>translation_length = <NUM_LIT:0><EOL>for (references, translation) in zip(reference_corpus,<EOL>translation_corpus):<EOL><INDENT>reference_length += min(len(r) for r in references)<EOL>translation_length += len(translation)<EOL>merged_ref_ngram_counts = collections.Counter()<EOL>for reference in references:<EOL><INDENT>merged_ref_ngram_counts |= _get_ngrams(reference, max_order)<EOL><DEDENT>translation_ngram_counts = _get_ngrams(translation, max_order)<EOL>overlap = translation_ngram_counts & merged_ref_ngram_counts<EOL>for ngram in overlap:<EOL><INDENT>matches_by_order[len(ngram)-<NUM_LIT:1>] += overlap[ngram]<EOL><DEDENT>for order in range(<NUM_LIT:1>, max_order+<NUM_LIT:1>):<EOL><INDENT>possible_matches = len(translation) - order + <NUM_LIT:1><EOL>if possible_matches > <NUM_LIT:0>:<EOL><INDENT>possible_matches_by_order[order-<NUM_LIT:1>] += possible_matches<EOL><DEDENT><DEDENT><DEDENT>precisions = [<NUM_LIT:0>] * max_order<EOL>for i in range(<NUM_LIT:0>, max_order):<EOL><INDENT>if smooth:<EOL><INDENT>precisions[i] = ((matches_by_order[i] + <NUM_LIT:1.>) /<EOL>(possible_matches_by_order[i] + <NUM_LIT:1.>))<EOL><DEDENT>else:<EOL><INDENT>if possible_matches_by_order[i] > <NUM_LIT:0>:<EOL><INDENT>precisions[i] = (float(matches_by_order[i]) /<EOL>possible_matches_by_order[i])<EOL><DEDENT>else:<EOL><INDENT>precisions[i] = <NUM_LIT:0.0><EOL><DEDENT><DEDENT><DEDENT>if min(precisions) > <NUM_LIT:0>:<EOL><INDENT>p_log_sum = sum((<NUM_LIT:1.> / max_order) * math.log(p) for p in precisions)<EOL>geo_mean = math.exp(p_log_sum)<EOL><DEDENT>else:<EOL><INDENT>geo_mean = <NUM_LIT:0><EOL><DEDENT>ratio = float(translation_length) / reference_length<EOL>if ratio > <NUM_LIT:1.0>:<EOL><INDENT>bp = <NUM_LIT:1.><EOL><DEDENT>else:<EOL><INDENT>bp = math.exp(<NUM_LIT:1> - <NUM_LIT:1.> / ratio)<EOL><DEDENT>bleu = geo_mean * bp<EOL>return (bleu, precisions, bp, ratio, translation_length, reference_length)<EOL>

Computes BLEU score of translated segments against one or more references.

    Args:
      reference_corpus: list of lists of references for each translation. Each
          reference should be tokenized into a list of tokens.
      translation_corpus: list of translations to score. Each translation
          should be tokenized into a list of tokens.
      max_order: Maximum n-gram order to use when computing BLEU score.
      smooth: Whether or not to apply Lin et al. 2004 smoothing.

    Returns:
      3-Tuple with the BLEU score, n-gram precisions, geometric mean of n-gram
      precisions and brevity penalty.


@register_metric('<STR_LIT>')<EOL>def squad_v2_exact_match(y_true: List[List[str]], y_predicted: List[str]) -> float:

EM_total = sum(normalize_answer(prediction) in map(normalize_answer, ground_truth)<EOL>for ground_truth, prediction in zip(y_true, y_predicted))<EOL>return <NUM_LIT:100> * EM_total / len(y_true) if len(y_true) > <NUM_LIT:0> else <NUM_LIT:0><EOL>

Calculates Exact Match score between y_true and y_predicted
        EM score uses the best matching y_true answer:
            if y_pred equal at least to one answer in y_true then EM = 1, else EM = 0

    The same as in SQuAD-v2.0

    Args:
        y_true: list of correct answers (correct answers are represented by list of strings)
        y_predicted: list of predicted answers

    Returns:
        exact match score : float


@register_metric('<STR_LIT>')<EOL>def squad_v1_exact_match(y_true: List[List[str]], y_predicted: List[str]) -> float:

EM_total = <NUM_LIT:0><EOL>count = <NUM_LIT:0><EOL>for ground_truth, prediction in zip(y_true, y_predicted):<EOL><INDENT>if len(ground_truth[<NUM_LIT:0>]) == <NUM_LIT:0>:<EOL><INDENT>continue<EOL><DEDENT>count += <NUM_LIT:1><EOL>EMs = [int(normalize_answer(gt) == normalize_answer(prediction)) for gt in ground_truth]<EOL>EM_total += max(EMs)<EOL><DEDENT>return <NUM_LIT:100> * EM_total / count if count > <NUM_LIT:0> else <NUM_LIT:0><EOL>

Calculates Exact Match score between y_true and y_predicted
        EM score uses the best matching y_true answer:
            if y_pred equal at least to one answer in y_true then EM = 1, else EM = 0
        Skips examples without an answer.
    Args:
        y_true: list of correct answers (correct answers are represented by list of strings)
        y_predicted: list of predicted answers
    Returns:
        exact match score : float


@register_metric('<STR_LIT>')<EOL>def squad_v2_f1(y_true: List[List[str]], y_predicted: List[str]) -> float:

f1_total = <NUM_LIT:0.0><EOL>for ground_truth, prediction in zip(y_true, y_predicted):<EOL><INDENT>prediction_tokens = normalize_answer(prediction).split()<EOL>f1s = []<EOL>for gt in ground_truth:<EOL><INDENT>gt_tokens = normalize_answer(gt).split()<EOL>if len(gt_tokens) == <NUM_LIT:0> or len(prediction_tokens) == <NUM_LIT:0>:<EOL><INDENT>f1s.append(float(gt_tokens == prediction_tokens))<EOL>continue<EOL><DEDENT>common = Counter(prediction_tokens) & Counter(gt_tokens)<EOL>num_same = sum(common.values())<EOL>if num_same == <NUM_LIT:0>:<EOL><INDENT>f1s.append(<NUM_LIT:0.0>)<EOL>continue<EOL><DEDENT>precision = <NUM_LIT:1.0> * num_same / len(prediction_tokens)<EOL>recall = <NUM_LIT:1.0> * num_same / len(gt_tokens)<EOL>f1 = (<NUM_LIT:2> * precision * recall) / (precision + recall)<EOL>f1s.append(f1)<EOL><DEDENT>f1_total += max(f1s)<EOL><DEDENT>return <NUM_LIT:100> * f1_total / len(y_true) if len(y_true) > <NUM_LIT:0> else <NUM_LIT:0><EOL>

Calculates F-1 score between y_true and y_predicted
        F-1 score uses the best matching y_true answer

    The same as in SQuAD-v2.0

    Args:
        y_true: list of correct answers (correct answers are represented by list of strings)
        y_predicted: list of predicted answers

    Returns:
        F-1 score : float


@register_metric('<STR_LIT>')<EOL>def squad_v1_f1(y_true: List[List[str]], y_predicted: List[str]) -> float:

f1_total = <NUM_LIT:0.0><EOL>count = <NUM_LIT:0><EOL>for ground_truth, prediction in zip(y_true, y_predicted):<EOL><INDENT>if len(ground_truth[<NUM_LIT:0>]) == <NUM_LIT:0>:<EOL><INDENT>continue<EOL><DEDENT>count += <NUM_LIT:1><EOL>prediction_tokens = normalize_answer(prediction).split()<EOL>f1s = []<EOL>for gt in ground_truth:<EOL><INDENT>gt_tokens = normalize_answer(gt).split()<EOL>common = Counter(prediction_tokens) & Counter(gt_tokens)<EOL>num_same = sum(common.values())<EOL>if num_same == <NUM_LIT:0>:<EOL><INDENT>f1s.append(<NUM_LIT:0.0>)<EOL>continue<EOL><DEDENT>precision = <NUM_LIT:1.0> * num_same / len(prediction_tokens)<EOL>recall = <NUM_LIT:1.0> * num_same / len(gt_tokens)<EOL>f1 = (<NUM_LIT:2> * precision * recall) / (precision + recall)<EOL>f1s.append(f1)<EOL><DEDENT>f1_total += max(f1s)<EOL><DEDENT>return <NUM_LIT:100> * f1_total / count if count > <NUM_LIT:0> else <NUM_LIT:0><EOL>

Calculates F-1 score between y_true and y_predicted
        F-1 score uses the best matching y_true answer

        Skips examples without an answer.
    Args:
        y_true: list of correct answers (correct answers are represented by list of strings)
        y_predicted: list of predicted answers
    Returns:
        F-1 score : float


def __call__(self, utterances_batch: list, history_batch: list,<EOL>states_batch: Optional[list]=None) -> Tuple[list, list]:

response = [random.choice(self.responses) for _ in utterances_batch]<EOL>if self.patterns is None:<EOL><INDENT>confidence = [self.default_confidence] * len(utterances_batch)<EOL><DEDENT>else:<EOL><INDENT>if self.ignore_case:<EOL><INDENT>utterances_batch = [utterance.lower() for utterance in utterances_batch]<EOL><DEDENT>if self.regex:<EOL><INDENT>confidence = [self.default_confidence*float(any([pattern.search(utterance) for pattern in self.patterns]))<EOL>for utterance in utterances_batch]<EOL><DEDENT>else:<EOL><INDENT>confidence = [self.default_confidence*float(any([pattern in utterance for pattern in self.patterns]))<EOL>for utterance in utterances_batch]<EOL><DEDENT><DEDENT>return response, confidence<EOL>

Returns skill inference result.

        Returns batches of skill inference results, estimated confidence
        levels and up to date states corresponding to incoming utterance
        batch.

        Args:
            utterances_batch: A batch of utterances of any type.
            history_batch: A batch of list typed histories for each utterance.
            states_batch: Optional. A batch of arbitrary typed states for
                each utterance.

        Returns:
            response: A batch of arbitrary typed skill inference results.
            confidence: A batch of float typed confidence levels for each of
                skill inference result.


def fit(self, data: List[Dict[Any, Any]]) -> None:

log.info(f"<STR_LIT>")<EOL>self.ec_data = [dict(item, **{<EOL>'<STR_LIT>': self.preprocess.spacy2dict(self.preprocess.analyze(item['<STR_LIT>'])),<EOL>'<STR_LIT>': self.preprocess.spacy2dict(self.preprocess.analyze(item['<STR_LIT>']+'<STR_LIT>'+item['<STR_LIT>']))<EOL>}) for item in data]<EOL>log.info('<STR_LIT>')<EOL>

Preprocess items `title` and `description` from the `data`

        Parameters:
            data: list of catalog items

        Returns:
            None


log.info(f"<STR_LIT>")<EOL>save_pickle(self.ec_data, self.save_path)<EOL>


log.info(f"<STR_LIT>")<EOL>for path in self.load_path:<EOL><INDENT>if Path.is_file(path):<EOL><INDENT>self.ec_data += load_pickle(path)<EOL><DEDENT>else:<EOL><INDENT>raise FileNotFoundError<EOL><DEDENT><DEDENT>log.info(f"<STR_LIT>")<EOL>


def __call__(self, queries: List[str], history: List[Any], states: List[Dict[Any, Any]]) ->Tuple[Tuple[List[Any], List[Any]], List[float], List[Any]]:

response: List = []<EOL>confidence: List = []<EOL>results_args: List = []<EOL>entropies: List = []<EOL>back_states: List = []<EOL>results_args_sim: List = []<EOL>log.debug(f"<STR_LIT>")<EOL>for item_idx, query in enumerate(queries):<EOL><INDENT>state = states[item_idx]<EOL>if isinstance(state, str):<EOL><INDENT>try:<EOL><INDENT>state = json.loads(state)<EOL><DEDENT>except:<EOL><INDENT>state = self.preprocess.parse_input(state)<EOL><DEDENT><DEDENT>if not state:<EOL><INDENT>state = {}<EOL><DEDENT>start = state['<STR_LIT:start>'] if '<STR_LIT:start>' in state else <NUM_LIT:0><EOL>stop = state['<STR_LIT>'] if '<STR_LIT>' in state else <NUM_LIT:5><EOL>state['<STR_LIT:start>'] = start<EOL>state['<STR_LIT>'] = stop<EOL>query = self.preprocess.analyze(query)<EOL>query, money_range = self.preprocess.extract_money(query)<EOL>log.debug(f"<STR_LIT>")<EOL>if len(money_range) == <NUM_LIT:2>:<EOL><INDENT>state['<STR_LIT>'] = money_range<EOL><DEDENT>score_title = [bleu_advanced(self.preprocess.lemmas(item['<STR_LIT>']),<EOL>self.preprocess.lemmas(self.preprocess.filter_nlp_title(query)),<EOL>weights = (<NUM_LIT:1>,), penalty = False) for item in self.ec_data]<EOL>score_feat = [bleu_advanced(self.preprocess.lemmas(item['<STR_LIT>']),<EOL>self.preprocess.lemmas(self.preprocess.filter_nlp(query)),<EOL>weights = (<NUM_LIT>, <NUM_LIT>), penalty = False) for idx, item in enumerate(self.ec_data)]<EOL>scores = np.mean([score_feat, score_title], axis=<NUM_LIT:0>).tolist()<EOL>scores_title = [(score, -len(self.ec_data[idx]['<STR_LIT>'])) for idx, score in enumerate(scores)]<EOL>raw_scores_ar = np.array(scores_title, dtype=[('<STR_LIT:x>', '<STR_LIT>'), ('<STR_LIT:y>', '<STR_LIT>')])<EOL>results_args = np.argsort(raw_scores_ar, order=('<STR_LIT:x>', '<STR_LIT:y>'))[::-<NUM_LIT:1>].tolist()<EOL>results_args_sim = [idx for idx in results_args if scores[idx] >= self.min_similarity]<EOL>log.debug(f"<STR_LIT>" +<EOL>f"<STR_LIT>")<EOL>results_args_sim = self._filter_state(state, results_args_sim)<EOL>results_args_sim_fil = [idx for idx in results_args_sim[start:stop]]<EOL>local_response = self._clean_items(results_args_sim_fil)<EOL>response.append(local_response)<EOL>confidence.append([(score_title[idx], score_feat[idx])<EOL>for idx in results_args_sim[start:stop]])<EOL>entropies.append(self._entropy_subquery(results_args_sim))<EOL>log.debug(f"<STR_LIT>")<EOL>back_states.append(state)<EOL><DEDENT>return (response, entropies), confidence, back_states<EOL>

Retrieve catalog items according to the BLEU measure

        Parameters:
            queries: list of queries
            history: list of previous queries
            states: list of dialog state

        Returns:
            response:   items:      list of retrieved items
                        entropies:  list of entropy attributes with corresponding values

            confidence: list of similarity scores
            state: dialog state


def _entropy_subquery(self, results_args: List[int]) -> List[Tuple[float, str, List[Tuple[str, int]]]]:

ent_fields: Dict = {}<EOL>for idx in results_args:<EOL><INDENT>for field in self.entropy_fields:<EOL><INDENT>if field in self.ec_data[idx]:<EOL><INDENT>if field not in ent_fields:<EOL><INDENT>ent_fields[field] = []<EOL><DEDENT>ent_fields[field].append(self.ec_data[idx][field].lower())<EOL><DEDENT><DEDENT><DEDENT>entropies = []<EOL>for key, value in ent_fields.items():<EOL><INDENT>count = Counter(value)<EOL>entropies.append(<EOL>(entropy(list(count.values()), base=<NUM_LIT:2>), key, count.most_common()))<EOL><DEDENT>entropies = sorted(entropies, key=itemgetter(<NUM_LIT:0>), reverse=True)<EOL>entropies = [<EOL>ent_item for ent_item in entropies if ent_item[<NUM_LIT:0>] >= self.min_entropy]<EOL>return entropies<EOL>

Calculate entropy of selected attributes for items from the catalog.

        Parameters:
            results_args: items id to consider

        Returns:
            entropies: entropy score with attribute name and corresponding values


self.x_train_features = vstack(list(query))<EOL>self.ec_data = data<EOL>


log.info("<STR_LIT>".format(self.save_path))<EOL>path = expand_path(self.save_path)<EOL>save_pickle((self.ec_data, self.x_train_features), path)<EOL>


log.info("<STR_LIT>".format(self.load_path))<EOL>self.ec_data, self.x_train_features = load_pickle(<EOL>expand_path(self.load_path))<EOL>


def __call__(self, q_vects: List[csr_matrix], histories: List[Any], states: List[Dict[Any, Any]]) -> Tuple[Tuple[List[Dict[Any, Any]], List[Any]], List[float], Dict[Any, Any]]:

log.info(f"<STR_LIT>")<EOL>if not isinstance(q_vects, list):<EOL><INDENT>q_vects = [q_vects]<EOL><DEDENT>if not isinstance(states, list):<EOL><INDENT>states = [states]<EOL><DEDENT>if not isinstance(histories, list):<EOL><INDENT>histories = [histories]<EOL><DEDENT>items: List = []<EOL>confidences: List = []<EOL>back_states: List = []<EOL>entropies: List = []<EOL>for idx, q_vect in enumerate(q_vects):<EOL><INDENT>log.info(f"<STR_LIT>")<EOL>if len(states) >= idx+<NUM_LIT:1>:<EOL><INDENT>state = states[idx]<EOL><DEDENT>else:<EOL><INDENT>state = {'<STR_LIT:start>': <NUM_LIT:0>, '<STR_LIT>': <NUM_LIT:5>}<EOL><DEDENT>if not isinstance(state, dict):<EOL><INDENT>state = {'<STR_LIT:start>': <NUM_LIT:0>, '<STR_LIT>': <NUM_LIT:5>}<EOL><DEDENT>if '<STR_LIT:start>' not in state:<EOL><INDENT>state['<STR_LIT:start>'] = <NUM_LIT:0><EOL><DEDENT>if '<STR_LIT>' not in state:<EOL><INDENT>state['<STR_LIT>'] = <NUM_LIT:5><EOL><DEDENT>if '<STR_LIT>' not in state:<EOL><INDENT>state['<STR_LIT>'] = []<EOL><DEDENT>log.info(f"<STR_LIT>")<EOL>if state['<STR_LIT>']:<EOL><INDENT>his_vect = self._list_to_csr(state['<STR_LIT>'][-<NUM_LIT:1>])<EOL>if not np.array_equal(his_vect.todense(), q_vect.todense()):<EOL><INDENT>q_comp = q_vect.maximum(his_vect)<EOL>complex_bool = self._take_complex_query(q_comp, q_vect)<EOL>log.info(f"<STR_LIT>")<EOL>if complex_bool is True:<EOL><INDENT>q_vect = q_comp<EOL>state['<STR_LIT:start>'] = <NUM_LIT:0><EOL>state['<STR_LIT>'] = <NUM_LIT:5><EOL><DEDENT>else:<EOL><INDENT>state['<STR_LIT>'] = []<EOL><DEDENT><DEDENT>else:<EOL><INDENT>log.info("<STR_LIT>")<EOL><DEDENT><DEDENT>else:<EOL><INDENT>log.info("<STR_LIT>")<EOL><DEDENT>state['<STR_LIT>'].append(self._csr_to_list(q_vect))<EOL>log.info(f"<STR_LIT>")<EOL>scores = self._similarity(q_vect)<EOL>answer_ids = np.argsort(scores)[::-<NUM_LIT:1>]<EOL>answer_ids = [idx for idx in answer_ids if scores[idx] >= self.min_similarity]<EOL>answer_ids = self._state_based_filter(answer_ids, state)<EOL>items.append([self.ec_data[idx]<EOL>for idx in answer_ids[state['<STR_LIT:start>']:state['<STR_LIT>']]])<EOL>confidences.append(<EOL>[scores[idx] for idx in answer_ids[state['<STR_LIT:start>']:state['<STR_LIT>']]])<EOL>back_states.append(state)<EOL>entropies.append(self._entropy_subquery(answer_ids))<EOL><DEDENT>return (items, entropies), confidences, back_states<EOL>

Retrieve catalog items according to the TFIDF measure

        Parameters:
            queries: list of queries
            history: list of previous queries
            states: list of dialog state

        Returns:
            response:   items:      list of retrieved items
                        entropies:  list of entropy attributes with corresponding values

            confidence: list of similarity scores
            state: dialog state


def _take_complex_query(self, q_prev: csr_matrix, q_cur: csr_matrix) -> bool:

prev_sim = self._similarity(q_prev)<EOL>cur_sim = self._similarity(q_cur)<EOL>log.debug(f"<STR_LIT>")<EOL>log.debug(f"<STR_LIT>")<EOL>if prev_sim.max() > cur_sim.max():<EOL><INDENT>return True<EOL><DEDENT>return False<EOL>

Decides whether to use the long compound query or the current short query

        Parameters:
            q_prev: previous query
            q_cur: current query

        Returns:
            Bool: whether to use the compound query


def _similarity(self, q_vect: Union[csr_matrix, List]) -> List[float]:

norm = sparse_norm(q_vect) * sparse_norm(self.x_train_features, axis=<NUM_LIT:1>)<EOL>cos_similarities = np.array(q_vect.dot(self.x_train_features.T).todense())/norm<EOL>cos_similarities = cos_similarities[<NUM_LIT:0>]<EOL>cos_similarities = np.nan_to_num(cos_similarities)<EOL>return cos_similarities<EOL>

Calculates cosine similarity between the user's query and product items.

        Parameters:
            q_cur: user's query

        Returns:
            cos_similarities: lits of similarity scores


def _state_based_filter(self, ids: List[int], state: Dict[Any, Any]):

for key, value in state.items():<EOL><INDENT>log.debug(f"<STR_LIT>")<EOL>if key in ['<STR_LIT>', '<STR_LIT:start>', '<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>continue<EOL><DEDENT>else:<EOL><INDENT>ids = [idx for idx in ids<EOL>if key in self.ec_data[idx]<EOL>if self.ec_data[idx][key].lower() == value.lower()]<EOL><DEDENT><DEDENT>return ids<EOL>

Filters the candidates based on the key-values from the state

        Parameters:
            ids: list of candidates
            state: dialog state

        Returns:
            ids: filtered list of candidates


ent_fields: Dict = {}<EOL>for idx in results_args:<EOL><INDENT>for field in self.entropy_fields:<EOL><INDENT>if field in self.ec_data[idx]:<EOL><INDENT>if field not in ent_fields:<EOL><INDENT>ent_fields[field] = []<EOL><DEDENT>ent_fields[field].append(self.ec_data[idx][field].lower())<EOL><DEDENT><DEDENT><DEDENT>entropies = []<EOL>for key, value in ent_fields.items():<EOL><INDENT>count = Counter(value)<EOL>entropies.append((entropy(list(count.values()), base=<NUM_LIT:2>), key, count.most_common()))<EOL><DEDENT>entropies = sorted(entropies, key=itemgetter(<NUM_LIT:0>), reverse=True)<EOL>entropies = [ent_item for ent_item in entropies if ent_item[<NUM_LIT:0>] >= self.min_entropy]<EOL>return entropies<EOL>


def __call__(self, utterances_batch: list, history_batch: list,<EOL>states_batch: Optional[list]=None) -> Tuple[list, list, list]:

batch_len = len(utterances_batch)<EOL>confidence_batch = [<NUM_LIT:1.0>] * batch_len<EOL>response_batch: List[Optional[str]] = [None] * batch_len<EOL>infer_indexes = []<EOL>if not states_batch:<EOL><INDENT>states_batch: List[Optional[dict]] = [None] * batch_len<EOL><DEDENT>for utt_i, utterance in enumerate(utterances_batch):<EOL><INDENT>if not states_batch[utt_i]:<EOL><INDENT>states_batch[utt_i] = {'<STR_LIT>': list(self.model.in_x), '<STR_LIT>': []}<EOL><DEDENT>if utterance:<EOL><INDENT>states_batch[utt_i]['<STR_LIT>'].pop(<NUM_LIT:0>)<EOL>states_batch[utt_i]['<STR_LIT>'].append(utterance)<EOL><DEDENT>if states_batch[utt_i]['<STR_LIT>']:<EOL><INDENT>response = self.proposal.format(states_batch[utt_i]['<STR_LIT>'][<NUM_LIT:0>])<EOL>response_batch[utt_i] = response<EOL><DEDENT>else:<EOL><INDENT>infer_indexes.append(utt_i)<EOL><DEDENT><DEDENT>if infer_indexes:<EOL><INDENT>infer_utterances = zip(*[tuple(states_batch[i]['<STR_LIT>']) for i in infer_indexes])<EOL>infer_results = self.model(*infer_utterances)<EOL>if len(self.model.out_params) > <NUM_LIT:1>:<EOL><INDENT>infer_results = ['<STR_LIT>'.join([str(out_y) for out_y in result]) for result in zip(*infer_results)]<EOL><DEDENT>for infer_i, infer_result in zip(infer_indexes, infer_results):<EOL><INDENT>response_batch[infer_i] = infer_result<EOL>states_batch[infer_i] = None<EOL><DEDENT><DEDENT>return response_batch, confidence_batch, states_batch<EOL>

Returns skill inference result.

        Returns batches of skill inference results, estimated confidence
            levels and up to date states corresponding to incoming utterance
            batch. Also handles interaction with multiargument models using
            skill states.

        Args:
            utterances_batch: A batch of utterances of any type.
            history_batch: Not used. A batch of list typed histories for each
                utterance.
            states_batch: A batch of states for each utterance.

        Returns:
            response: A batch of arbitrary typed skill inference results.
            confidence: A batch of float typed confidence levels for each of
                skill inference result.
            states: Optional. A batch of states for each response.


unit = np.eye(k, dtype=int)<EOL>return unit[x]<EOL>

Takes an array of integers and transforms it
to an array of one-hot encoded vectors


<EOL>if dropout == <NUM_LIT:0.0>:<EOL><INDENT>return inputs<EOL><DEDENT>inputs_func = lambda x: kb.ones_like(inputs[:, :, <NUM_LIT:0>:<NUM_LIT:1>])<EOL>inputs_mask = kl.Lambda(inputs_func)(inputs)<EOL>inputs_mask = kl.Dropout(dropout)(inputs_mask)<EOL>tiling_shape = [<NUM_LIT:1>, <NUM_LIT:1>, kb.shape(inputs)[<NUM_LIT:2>]] + [<NUM_LIT:1>] * (kb.ndim(inputs) - <NUM_LIT:3>)<EOL>inputs_mask = kl.Lambda(kb.tile, arguments={"<STR_LIT:n>": tiling_shape},<EOL>output_shape=inputs._keras_shape[<NUM_LIT:1>:])(inputs_mask)<EOL>answer = kl.Multiply()([inputs, inputs_mask])<EOL>return answer<EOL>

Drops with :dropout probability temporal steps of input 3D tensor


position_inputs = kb.cumsum(kb.ones_like(inputs, dtype="<STR_LIT>"), axis=<NUM_LIT:1>)<EOL>position_inputs *= kb.cast(kb.not_equal(inputs, pad), "<STR_LIT>")<EOL>return kb.log(<NUM_LIT:1.0> + position_inputs)<EOL>

A layer filling i-th column of a 2D tensor with
1+ln(1+i) when it contains a meaningful symbol
and with 0 when it contains PAD


def predict_with_model(config_path: [Path, str]) -> List[Optional[List[str]]]:

config = parse_config(config_path)<EOL>reader_config = config['<STR_LIT>']<EOL>reader = get_model(reader_config['<STR_LIT>'])()<EOL>data_path = expand_path(reader_config.get('<STR_LIT>', '<STR_LIT>'))<EOL>read_params = {k: v for k, v in reader_config.items() if k not in ['<STR_LIT>', '<STR_LIT>']}<EOL>data: Dict = reader.read(data_path, **read_params)<EOL>iterator_config = config['<STR_LIT>']<EOL>iterator: MorphoTaggerDatasetIterator = from_params(iterator_config, data=data)<EOL>model = build_model(config, load_trained=True)<EOL>answers = [None] * len(iterator.test)<EOL>batch_size = config['<STR_LIT>'].get("<STR_LIT>", -<NUM_LIT:1>)<EOL>for indexes, (x, _) in iterator.gen_batches(<EOL>batch_size=batch_size, data_type="<STR_LIT:test>", shuffle=False, return_indexes=True):<EOL><INDENT>y = model(x)<EOL>for i, elem in zip(indexes, y):<EOL><INDENT>answers[i] = elem<EOL><DEDENT><DEDENT>outfile = config['<STR_LIT>'].get("<STR_LIT>")<EOL>if outfile is not None:<EOL><INDENT>outfile = Path(outfile)<EOL>if not outfile.exists():<EOL><INDENT>outfile.parent.mkdir(parents=True, exist_ok=True)<EOL><DEDENT>with open(outfile, "<STR_LIT:w>", encoding="<STR_LIT:utf8>") as fout:<EOL><INDENT>for elem in answers:<EOL><INDENT>fout.write(elem + "<STR_LIT:\n>")<EOL><DEDENT><DEDENT><DEDENT>return answers<EOL>

Returns predictions of morphotagging model given in config :config_path:.

    Args:
        config_path: a path to config

    Returns:
        a list of morphological analyses for each sentence. Each analysis is either a list of tags
        or a list of full CONLL-U descriptions.


def set_format_mode(self, format_mode: str = "<STR_LIT>") -> None:

self.format_mode = format_mode<EOL>self._make_format_string()<EOL>

A function that sets format for output and recalculates `self.format_string`.

        Args:
            format_mode: output format,
                in `basic` mode output data contains 4 columns (id, word, pos, features),
                in `conllu` or `ud` mode it contains 10 columns:
                id, word, lemma, pos, xpos, feats, head, deprel, deps, misc
                (see http://universaldependencies.org/format.html for details)
                Only id, word, tag and pos values are present in current version,
                other columns are filled by `_` value.

        Returns:


def __call__(self, X: List[List[str]], Y: List[List[str]]) -> List[Union[List[str], str]]:

return [self.prettify(x, y) for x, y in zip(X, Y)]<EOL>

Calls the ``prettify`` function for each input sentence.

        Args:
            X: a list of input sentences
            Y: a list of list of tags for sentence words

        Returns:
            a list of prettified morphological analyses


def prettify(self, tokens: List[str], tags: List[str]) -> Union[List[str], str]:

answer = []<EOL>for i, (word, tag) in enumerate(zip(tokens, tags)):<EOL><INDENT>answer.append(self.format_string.format(i + <NUM_LIT:1>, word, *make_pos_and_tag(tag)))<EOL><DEDENT>if self.return_string:<EOL><INDENT>answer = self.begin + self.sep.join(answer) + self.end<EOL><DEDENT>return answer<EOL>

Prettifies output of morphological tagger.

        Args:
            tokens: tokenized source sentence
            tags: list of tags, the output of a tagger

        Returns:
            the prettified output of the tagger.

        Examples:
            >>> sent = "John really likes pizza .".split()
            >>> tags = ["PROPN,Number=Sing", "ADV",
            >>>         "VERB,Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin",
            >>>         "NOUN,Number=Sing", "PUNCT"]
            >>> prettifier = TagOutputPrettifier(mode='basic')
            >>> self.prettify(sent, tags)
                1	John	PROPN	Number=Sing
                2	really	ADV	_
                3	likes	VERB	Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin
                4	pizza	NOUN	Number=Sing
                5	.	PUNCT	_
            >>> prettifier = TagOutputPrettifier(mode='ud')
            >>> self.prettify(sent, tags)
                1	John	_	PROPN	_	Number=Sing	_	_	_	_
                2	really	_	ADV	_	_	_	_	_	_
                3	likes	_	VERB	_	Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin	_	_	_	_
                4	pizza	_	NOUN	_	Number=Sing	_	_	_	_
                5	.	_	PUNCT	_	_	_	_	_	_


@property<EOL><INDENT>def symbols_number_(self) -> int:<DEDENT>

return len(self.symbols)<EOL>


@property<EOL><INDENT>def tags_number_(self) -> int:<DEDENT>

return len(self.tags)<EOL>


word_inputs = kl.Input(shape=(None, MAX_WORD_LENGTH+<NUM_LIT:2>), dtype="<STR_LIT>")<EOL>inputs = [word_inputs]<EOL>word_outputs = self._build_word_cnn(word_inputs)<EOL>if len(self.word_vectorizers) > <NUM_LIT:0>:<EOL><INDENT>additional_word_inputs = [kl.Input(shape=(None, input_dim), dtype="<STR_LIT>")<EOL>for input_dim, dense_dim in self.word_vectorizers]<EOL>inputs.extend(additional_word_inputs)<EOL>additional_word_embeddings = [kl.Dense(dense_dim)(additional_word_inputs[i])<EOL>for i, (_, dense_dim) in enumerate(self.word_vectorizers)]<EOL>word_outputs = kl.Concatenate()([word_outputs] + additional_word_embeddings)<EOL><DEDENT>outputs, lstm_outputs = self._build_basic_network(word_outputs)<EOL>compile_args = {"<STR_LIT>": ko.nadam(lr=<NUM_LIT>, clipnorm=<NUM_LIT>),<EOL>"<STR_LIT>": "<STR_LIT>", "<STR_LIT>": ["<STR_LIT>"]}<EOL>self.model_ = Model(inputs, outputs)<EOL>self.model_.compile(**compile_args)<EOL>if self.verbose > <NUM_LIT:0>:<EOL><INDENT>self.model_.summary(print_fn=log.info)<EOL><DEDENT>return self<EOL>


inputs = kl.Lambda(kb.one_hot, arguments={"<STR_LIT>": self.symbols_number_},<EOL>output_shape=lambda x: tuple(x) + (self.symbols_number_,))(inputs)<EOL>char_embeddings = kl.Dense(self.char_embeddings_size, use_bias=False)(inputs)<EOL>conv_outputs = []<EOL>self.char_output_dim_ = <NUM_LIT:0><EOL>for window_size, filters_number in zip(self.char_window_size, self.char_filters):<EOL><INDENT>curr_output = char_embeddings<EOL>curr_filters_number = (min(self.char_filter_multiple * window_size, <NUM_LIT:200>)<EOL>if filters_number is None else filters_number)<EOL>for _ in range(self.char_conv_layers - <NUM_LIT:1>):<EOL><INDENT>curr_output = kl.Conv2D(curr_filters_number, (<NUM_LIT:1>, window_size),<EOL>padding="<STR_LIT>", activation="<STR_LIT:relu>",<EOL>data_format="<STR_LIT>")(curr_output)<EOL>if self.conv_dropout > <NUM_LIT:0.0>:<EOL><INDENT>curr_output = kl.Dropout(self.conv_dropout)(curr_output)<EOL><DEDENT><DEDENT>curr_output = kl.Conv2D(curr_filters_number, (<NUM_LIT:1>, window_size),<EOL>padding="<STR_LIT>", activation="<STR_LIT:relu>",<EOL>data_format="<STR_LIT>")(curr_output)<EOL>conv_outputs.append(curr_output)<EOL>self.char_output_dim_ += curr_filters_number<EOL><DEDENT>if len(conv_outputs) > <NUM_LIT:1>:<EOL><INDENT>conv_output = kl.Concatenate(axis=-<NUM_LIT:1>)(conv_outputs)<EOL><DEDENT>else:<EOL><INDENT>conv_output = conv_outputs[<NUM_LIT:0>]<EOL><DEDENT>highway_input = kl.Lambda(kb.max, arguments={"<STR_LIT>": -<NUM_LIT:2>})(conv_output)<EOL>if self.intermediate_dropout > <NUM_LIT:0.0>:<EOL><INDENT>highway_input = kl.Dropout(self.intermediate_dropout)(highway_input)<EOL><DEDENT>for i in range(self.char_highway_layers - <NUM_LIT:1>):<EOL><INDENT>highway_input = Highway(activation="<STR_LIT:relu>")(highway_input)<EOL>if self.highway_dropout > <NUM_LIT:0.0>:<EOL><INDENT>highway_input = kl.Dropout(self.highway_dropout)(highway_input)<EOL><DEDENT><DEDENT>highway_output = Highway(activation="<STR_LIT:relu>")(highway_input)<EOL>return highway_output<EOL>


if self.word_dropout > <NUM_LIT:0.0>:<EOL><INDENT>lstm_outputs = kl.Dropout(self.word_dropout)(word_outputs)<EOL><DEDENT>else:<EOL><INDENT>lstm_outputs = word_outputs<EOL><DEDENT>for j in range(self.word_lstm_layers-<NUM_LIT:1>):<EOL><INDENT>lstm_outputs = kl.Bidirectional(<EOL>kl.LSTM(self.word_lstm_units[j], return_sequences=True,<EOL>dropout=self.lstm_dropout))(lstm_outputs)<EOL><DEDENT>lstm_outputs = kl.Bidirectional(<EOL>kl.LSTM(self.word_lstm_units[-<NUM_LIT:1>], return_sequences=True,<EOL>dropout=self.lstm_dropout))(lstm_outputs)<EOL>pre_outputs = kl.TimeDistributed(<EOL>kl.Dense(self.tags_number_, activation="<STR_LIT>",<EOL>activity_regularizer=self.regularizer),<EOL>name="<STR_LIT:p>")(lstm_outputs)<EOL>return pre_outputs, lstm_outputs<EOL>

Creates the basic network architecture,
transforming word embeddings to intermediate outputs


def train_on_batch(self, data: List[Iterable], labels: Iterable[list]) -> None:

X, Y = self._transform_batch(data, labels)<EOL>self.model_.train_on_batch(X, Y)<EOL>

Trains model on a single batch

        Args:
            data: a batch of word sequences
            labels: a batch of correct tag sequences
        Returns:
            the trained model


def predict_on_batch(self, data: Union[list, tuple],<EOL>return_indexes: bool = False) -> List[List[str]]:

X = self._transform_batch(data)<EOL>objects_number, lengths = len(X[<NUM_LIT:0>]), [len(elem) for elem in data[<NUM_LIT:0>]]<EOL>Y = self.model_.predict_on_batch(X)<EOL>labels = np.argmax(Y, axis=-<NUM_LIT:1>)<EOL>answer: List[List[str]] = [None] * objects_number<EOL>for i, (elem, length) in enumerate(zip(labels, lengths)):<EOL><INDENT>elem = elem[:length]<EOL>answer[i] = elem if return_indexes else self.tags.idxs2toks(elem)<EOL><DEDENT>return answer<EOL>

Makes predictions on a single batch

Args:
    data: a batch of word sequences together with additional inputs
    return_indexes: whether to return tag indexes in vocabulary or tags themselves

Returns:
    a batch of label sequences


def _make_sent_vector(self, sent: List, bucket_length: int =None) -> np.ndarray:

bucket_length = bucket_length or len(sent)<EOL>answer = np.zeros(shape=(bucket_length, MAX_WORD_LENGTH+<NUM_LIT:2>), dtype=np.int32)<EOL>for i, word in enumerate(sent):<EOL><INDENT>answer[i, <NUM_LIT:0>] = self.tags.tok2idx("<STR_LIT>")<EOL>m = min(len(word), MAX_WORD_LENGTH)<EOL>for j, x in enumerate(word[-m:]):<EOL><INDENT>answer[i, j+<NUM_LIT:1>] = self.symbols.tok2idx(x)<EOL><DEDENT>answer[i, m+<NUM_LIT:1>] = self.tags.tok2idx("<STR_LIT>")<EOL>answer[i, m+<NUM_LIT:2>:] = self.tags.tok2idx("<STR_LIT>")<EOL><DEDENT>return answer<EOL>

Transforms a sentence to Numpy array, which will be the network input.

        Args:
            sent: input sentence
            bucket_length: the width of the bucket

        Returns:
            A 3d array, answer[i][j][k] contains the index of k-th letter
            in j-th word of i-th input sentence.


def _make_tags_vector(self, tags, bucket_length=None) -> np.ndarray:

bucket_length = bucket_length or len(tags)<EOL>answer = np.zeros(shape=(bucket_length,), dtype=np.int32)<EOL>for i, tag in enumerate(tags):<EOL><INDENT>answer[i] = self.tags.tok2idx(tag)<EOL><DEDENT>return answer<EOL>

Transforms a sentence of tags to Numpy array, which will be the network target.

        Args:
            tags: input sentence of tags
            bucket_length: the width of the bucket

        Returns:
            A 2d array, answer[i][j] contains the index of j-th tag in i-th input sentence.


Saves model weights to a file

        Args:
            outfile: file with model weights (other model components should be given in config)


Loads model weights from a file

        Args:
            infile: file to load model weights from


def __init__(self,<EOL>population_size: int,<EOL>p_crossover: float = <NUM_LIT:0.5>, crossover_power: float = <NUM_LIT:0.5>,<EOL>p_mutation: float = <NUM_LIT:0.5>, mutation_power: float = <NUM_LIT:0.1>,<EOL>key_main_model: str = "<STR_LIT>",<EOL>seed: int = None,<EOL>train_partition: int = <NUM_LIT:1>,<EOL>elitism_with_weights: bool = False,<EOL>prefix: str = "<STR_LIT>",<EOL>models_path_variable: str = "<STR_LIT>",<EOL>**kwargs):

super().__init__(prefix=prefix, seed=seed, **kwargs)<EOL>self.main_model_path = list(self.find_model_path(self.basic_config, key_main_model))[<NUM_LIT:0>]<EOL>log.info("<STR_LIT>".format(self.main_model_path))<EOL>self.population_size = population_size<EOL>self.p_crossover = p_crossover<EOL>self.p_mutation = p_mutation<EOL>self.mutation_power = mutation_power<EOL>self.crossover_power = crossover_power<EOL>self.elitism_with_weights = elitism_with_weights<EOL>self.n_saved_best_pretrained = <NUM_LIT:0><EOL>self.train_partition = train_partition<EOL>self.evolution_model_id = <NUM_LIT:0><EOL>self.basic_config, self.models_path = self.remove_key_from_config(<EOL>self.basic_config, ["<STR_LIT>", "<STR_LIT>", models_path_variable])<EOL>self.models_path = Path(self.models_path)<EOL>for path_name in ["<STR_LIT>", "<STR_LIT>"]:<EOL><INDENT>occured_mpaths = list(self.find_model_path(self.basic_config, path_name))<EOL>for ppath in occured_mpaths:<EOL><INDENT>new_path = self.get_value_from_config(<EOL>self.basic_config,<EOL>ppath + [path_name]).replace(models_path_variable, "<STR_LIT>" + path_name.upper())<EOL>self.insert_value_or_dict_into_config(self.basic_config, ppath + [path_name], new_path)<EOL><DEDENT><DEDENT>self.path_to_models_save_path = ["<STR_LIT>", "<STR_LIT>", "<STR_LIT>"]<EOL>self.path_to_models_load_path = ["<STR_LIT>", "<STR_LIT>", "<STR_LIT>"]<EOL>self.insert_value_or_dict_into_config(self.basic_config, self.path_to_models_save_path, str(self.models_path))<EOL>self.insert_value_or_dict_into_config(self.basic_config, self.path_to_models_load_path, str(self.models_path))<EOL>try:<EOL><INDENT>self.evolve_metric_optimization = self.get_value_from_config(<EOL>self.basic_config, list(self.find_model_path(<EOL>self.basic_config, "<STR_LIT>"))[<NUM_LIT:0>] + ["<STR_LIT>"])<EOL><DEDENT>except:<EOL><INDENT>self.evolve_metric_optimization = "<STR_LIT>"<EOL><DEDENT>

Initialize evolution with random population


def first_generation(self, iteration: int = <NUM_LIT:0>) -> List[dict]:

population = []<EOL>for i in range(self.population_size):<EOL><INDENT>config = self.initialize_params_in_config(self.basic_config, self.paths_to_params)<EOL>self.insert_value_or_dict_into_config(config, self.path_to_models_save_path,<EOL>str(self.models_path / f"<STR_LIT>" / f"<STR_LIT>"))<EOL>self.insert_value_or_dict_into_config(config, self.path_to_models_load_path,<EOL>str(self.models_path / f"<STR_LIT>" / f"<STR_LIT>"))<EOL>config["<STR_LIT>"] = self.evolution_model_id<EOL>self.evolution_model_id += <NUM_LIT:1><EOL>population.append(config)<EOL><DEDENT>return population<EOL>

Initialize first generation randomly according to the given constraints is self.params

Args:
    iteration: number of iteration

Returns:
    first generation that consists of self.population_size individuums


def next_generation(self, generation: List[dict], scores: List[float], iteration: int) -> List[dict]:

next_population = self.selection_of_best_with_weights(generation, scores)<EOL>log.info("<STR_LIT>".format(self.n_saved_best_pretrained))<EOL>offsprings = self.crossover(generation, scores)<EOL>changable_next = self.mutation(offsprings)<EOL>next_population.extend(changable_next)<EOL>for i in range(self.n_saved_best_pretrained):<EOL><INDENT>if self.train_partition != <NUM_LIT:1>:<EOL><INDENT>file_ext = str(Path(next_population[i]["<STR_LIT>"]["<STR_LIT:train>"]).suffix)<EOL>next_population[i]["<STR_LIT>"]["<STR_LIT:train>"] = "<STR_LIT:_>".join(<EOL>Path(next_population[i]["<STR_LIT>"]["<STR_LIT:train>"]).stem.split("<STR_LIT:_>")[:-<NUM_LIT:1>]<EOL>) + "<STR_LIT:_>" + str(iteration % self.train_partition) + file_ext<EOL><DEDENT>if self.elitism_with_weights:<EOL><INDENT>self.insert_value_or_dict_into_config(<EOL>next_population[i], self.path_to_models_load_path,<EOL>self.get_value_from_config(next_population[i], self.path_to_models_save_path))<EOL><DEDENT>else:<EOL><INDENT>self.insert_value_or_dict_into_config(<EOL>next_population[i], self.path_to_models_load_path,<EOL>str(self.models_path / f"<STR_LIT>" / f"<STR_LIT>"))<EOL><DEDENT>self.insert_value_or_dict_into_config(<EOL>next_population[i], self.path_to_models_save_path,<EOL>str(self.models_path / f"<STR_LIT>" / f"<STR_LIT>"))<EOL><DEDENT>for i in range(self.n_saved_best_pretrained, self.population_size):<EOL><INDENT>if self.train_partition != <NUM_LIT:1>:<EOL><INDENT>file_ext = str(Path(next_population[i]["<STR_LIT>"]["<STR_LIT:train>"]).suffix)<EOL>next_population[i]["<STR_LIT>"]["<STR_LIT:train>"] = "<STR_LIT:_>".join(<EOL>[str(p) for p in Path(next_population[i]["<STR_LIT>"]["<STR_LIT:train>"]).stem.split("<STR_LIT:_>")[:-<NUM_LIT:1>]])+ "<STR_LIT:_>" + str(iteration % self.train_partition) + file_ext<EOL><DEDENT>self.insert_value_or_dict_into_config(<EOL>next_population[i], self.path_to_models_save_path,<EOL>str(self.models_path / f"<STR_LIT>" / f"<STR_LIT>"))<EOL>self.insert_value_or_dict_into_config(<EOL>next_population[i], self.path_to_models_load_path,<EOL>str(self.models_path / f"<STR_LIT>" / f"<STR_LIT>"))<EOL>next_population[i]["<STR_LIT>"] = self.evolution_model_id<EOL>self.evolution_model_id += <NUM_LIT:1><EOL><DEDENT>return next_population<EOL>

Provide replacement

Args:
    generation: current generation (set of self.population_size configs
    scores: corresponding scores that should be maximized
    iteration: iteration number

Returns:
    the next generation according to the given scores of current generation


def selection_of_best_with_weights(self, population: List[dict], scores: List[float]) -> List[dict]:

ranges = self.range_scores(scores)<EOL>a = <NUM_LIT:1.> / (<NUM_LIT:1.> - self.population_size)<EOL>b = self.population_size / (self.population_size - <NUM_LIT:1.>)<EOL>probas_to_be_selected = a * ranges + b<EOL>selected = []<EOL>for i in range(self.population_size):<EOL><INDENT>if self.decision(probas_to_be_selected[i]):<EOL><INDENT>selected.append(deepcopy(population[i]))<EOL><DEDENT><DEDENT>self.n_saved_best_pretrained = len(selected)<EOL>return selected<EOL>

Select individuums to save with weights for the next generation from given population.
Range is an order of an individuum within sorted scores (1 range = max-score, self.population_size = min-score)
Individuum with the best score has probability equal to 1 (100%).
Individuum with the worst score has probability equal to 0 (0%).
Probability of i-th individuum to be selected with weights is (a * range_i + b)
where a = 1. / (1. - self.population_size), and
b = self.population_size / (self.population_size - 1.)

Args:
    population: self.population_size individuums
    scores: list of corresponding scores

Returns:
    selected self.n_saved_best_pretrained (changable) individuums


def range_scores(self, scores: List[float]) -> np.ndarray:

not_none_scores = np.array([x for x in scores if x is not None])<EOL>if len(not_none_scores) == <NUM_LIT:0>:<EOL><INDENT>not_none_scores = np.array([<NUM_LIT:0>])<EOL><DEDENT>min_score = np.min(not_none_scores)<EOL>max_score = np.max(not_none_scores)<EOL>for i in range(self.population_size):<EOL><INDENT>if scores[i] is None:<EOL><INDENT>if self.evolve_metric_optimization == "<STR_LIT>":<EOL><INDENT>scores[i] = min_score - self.eps<EOL><DEDENT>else:<EOL><INDENT>scores[i] = max_score + self.eps<EOL><DEDENT><DEDENT><DEDENT>scores = np.array(scores, dtype='<STR_LIT:float>')<EOL>sorted_ids = np.argsort(scores)<EOL>if self.evolve_metric_optimization == "<STR_LIT>":<EOL><INDENT>sorted_ids = sorted_ids[::-<NUM_LIT:1>]<EOL><DEDENT>ranges = np.array([self.population_size - np.where(i == sorted_ids)[<NUM_LIT:0>][<NUM_LIT:0>]<EOL>for i in np.arange(self.population_size)])<EOL>return ranges<EOL>

Ranges scores,
range 1 corresponds to the best score,
range self.population_size corresponds to the worst score.

Args:
    scores: list of corresponding scores of population

Returns:
    ranges


def crossover(self, population: List[dict], scores: List[float]) -> List[dict]:

offsprings = []<EOL>ranges = self.range_scores(scores)<EOL>a = <NUM_LIT:1.> / (<NUM_LIT:1.> - self.population_size)<EOL>b = self.population_size / (self.population_size - <NUM_LIT:1.>)<EOL>probas_to_be_parent = (a * ranges + b) / np.sum(a * ranges + b)<EOL>intervals = np.array([np.sum(probas_to_be_parent[:i]) for i in range(self.population_size)])<EOL>for i in range(self.population_size - self.n_saved_best_pretrained):<EOL><INDENT>rs = np.random.random(<NUM_LIT:2>)<EOL>parents = population[np.where(rs[<NUM_LIT:0>] > intervals)[<NUM_LIT:0>][-<NUM_LIT:1>]], population[np.where(rs[<NUM_LIT:1>] > intervals)[<NUM_LIT:0>][-<NUM_LIT:1>]]<EOL>if self.decision(self.p_crossover):<EOL><INDENT>params_perm = np.random.permutation(self.n_params)<EOL>curr_offsprings = [deepcopy(parents[<NUM_LIT:0>]),<EOL>deepcopy(parents[<NUM_LIT:1>])]<EOL>part = int(self.crossover_power * self.n_params)<EOL>for j in range(self.n_params - part, self.n_params):<EOL><INDENT>self.insert_value_or_dict_into_config(curr_offsprings[<NUM_LIT:0>],<EOL>self.paths_to_params[<EOL>params_perm[j]],<EOL>self.get_value_from_config(<EOL>parents[<NUM_LIT:1>],<EOL>self.paths_to_params[<EOL>params_perm[j]]))<EOL>self.insert_value_or_dict_into_config(curr_offsprings[<NUM_LIT:1>],<EOL>self.paths_to_params[<EOL>params_perm[j]],<EOL>self.get_value_from_config(<EOL>parents[<NUM_LIT:0>],<EOL>self.paths_to_params[<EOL>params_perm[j]]))<EOL><DEDENT>offsprings.append(deepcopy(curr_offsprings[<NUM_LIT:0>]))<EOL><DEDENT>else:<EOL><INDENT>offsprings.append(deepcopy(parents[<NUM_LIT:0>]))<EOL><DEDENT><DEDENT>return offsprings<EOL>

Recombine randomly population in pairs and cross over them with given probability.
Cross over from two parents produces two offsprings
each of which contains crossover_power portion of the parameter values from one parent,
 and the other (1 - crossover_power portion) from the other parent

Args:
    population: self.population_size individuums
    scores: list of corresponding scores

Returns:
    (self.population_size - self.n_saved_best_pretained) offsprings


def mutation(self, population: List[dict]) -> List[dict]:

mutated = []<EOL>for individuum in population:<EOL><INDENT>mutated_individuum = deepcopy(individuum)<EOL>for path_ in self.paths_to_params:<EOL><INDENT>param_value = self.get_value_from_config(individuum, path_)<EOL>self.insert_value_or_dict_into_config(<EOL>mutated_individuum, path_,<EOL>self.mutation_of_param(path_, param_value))<EOL><DEDENT>mutated.append(mutated_individuum)<EOL><DEDENT>return mutated<EOL>

Mutate each parameter of each individuum in population

Args:
    population: self.population_size individuums

Returns:
    mutated population