jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def delete_user_session(user_id, session_key):
    try:
        log.debug("Deleted session_key: " + session_key)
        session.query(User_Sessions).filter(User_Sessions.user_id==user_id,
                                            User_Sessions.session_key==session_key).delete()
        session.commit()
    except (exc.OperationalError, exc.InvalidRequestError):
        session.rollback()
        log.exception(e)
//  [python]
def delete_user_session(user_id, session_key):
    try:
        log.info("Deleted session_key : " + session_key)
        session.query(User_Sessions).filter(User_Sessions.user_id==user_id,
                                            User_Sessions.session_key==session_key).delete()
        session.commit()
    except (exc.OperationalError, exc.InvalidRequestError):
        session.rollback()
        log.exception(e)
//  [python]
def test_can_read_token_from_headers(self):
    """Tests that Sydent correctly extracts an auth token from request headers"""
    self.sydent.run()

    request, _ = make_request(
        self.sydent.reactor, "GET", "/_matrix/identity/v2/hash_details"
    )
    request.requestHeaders.addRawHeader(
        b"Authorization", b"Bearer " + self.test_token.encode("ascii")
    )

    token = tokenFromRequest(request)

    self.assertEqual(token, self.test_token)
//  [python]
def test_can_read_token_from_headers(self):
    """Tests that Sydent correct extracts an auth token from request headers"""
    self.sydent.run()

    request, _ = make_request(
        self.sydent.reactor, "GET", "/_matrix/identity/v2/hash_details"
    )
    request.requestHeaders.addRawHeader(
        b"Authorization", b"Bearer " + self.test_token.encode("ascii")
    )

    token = tokenFromRequest(request)

    self.assertEqual(token, self.test_token)
//  [python]
def save_cover_from_url(url, book_path):
    try:
        if not cli.allow_localhost:
            # 127.0.x.x, localhost, [::1], [::ffff:7f00:1]
            ip = socket.getaddrinfo(urlparse(url).hostname, 0)[0][4][0]
            if ip.startswith("127.") or ip.startswith('::ffff:7f') or ip == "::1" or ip == "0.0.0.0" or ip == "::":
                log.error("Localhost was accessed for cover upload")
                return False, _("You are not allowed to access localhost for cover uploads")
        img = requests.get(url, timeout=(10, 200), allow_redirects=False)      # ToDo: Error Handling
        img.raise_for_status()
        return save_cover(img, book_path)
    except (socket.gaierror,
            requests.exceptions.HTTPError,
            requests.exceptions.ConnectionError,
            requests.exceptions.Timeout) as ex:
        log.info(u'Cover Download Error %s', ex)
        return False, _("Error Downloading Cover")
    except MissingDelegateError as ex:
        log.info(u'File Format Error %s', ex)
        return False, _("Cover Format Error")
//  [python]
def save_cover_from_url(url, book_path):
    try:
        if not cli.allow_localhost:
            # 127.0.x.x, localhost, [::1], [::ffff:7f00:1]
            ip = socket.getaddrinfo(urlparse(url).hostname, 0)[0][4][0]
            if ip.startswith("127.") or ip.startswith('::ffff:7f') or ip == "::1":
                log.error("Localhost was accessed for cover upload")
                return False, _("You are not allowed to access localhost for cover uploads")
        img = requests.get(url, timeout=(10, 200))      # ToDo: Error Handling
        img.raise_for_status()
        return save_cover(img, book_path)
    except (socket.gaierror,
            requests.exceptions.HTTPError,
            requests.exceptions.ConnectionError,
            requests.exceptions.Timeout) as ex:
        log.info(u'Cover Download Error %s', ex)
        return False, _("Error Downloading Cover")
    except MissingDelegateError as ex:
        log.info(u'File Format Error %s', ex)
        return False, _("Cover Format Error")
//  [python]
def store_user_session():
    if flask_session.get('_user_id', ""):
        try:
            if not check_user_session(flask_session.get('_user_id', ""), flask_session.get('_id', "")):
                user_session = User_Sessions(flask_session.get('_user_id', ""), flask_session.get('_id', ""))
                session.add(user_session)
                session.commit()
                log.debug("Login and store session : " + flask_session.get('_id', ""))
            else:
                log.debug("Found stored session: " + flask_session.get('_id', ""))
        except (exc.OperationalError, exc.InvalidRequestError) as e:
            session.rollback()
            log.exception(e)
    else:
        log.error("No user id in session")
//  [python]
def store_user_session():
    if flask_session.get('_user_id', ""):
        try:
            if not check_user_session(flask_session.get('_user_id', ""), flask_session.get('_id', "")):
                user_session = User_Sessions(flask_session.get('_user_id', ""), flask_session.get('_id', ""))
                session.add(user_session)
                session.commit()
                log.info("Login and store session : " + flask_session.get('_id', ""))
            else:
                log.info("Found stored session : " + flask_session.get('_id', ""))
        except (exc.OperationalError, exc.InvalidRequestError) as e:
            session.rollback()
            log.exception(e)
    else:
        log.error("No user id in session")
//  [python]
def handle(self, command, kwargs=None):
    """
    Dispatch and handle processing of the given command.

    :param command: Name of command to run.
    :type command: unicode
    :param kwargs: Arguments to pass to the command handler. If empty, `request.POST` is used.
    :type kwargs: dict
    :return: response.
    :rtype: HttpResponse
    """

    kwargs = kwargs or dict(six.iteritems(self.request.POST))
    try:
        handler = self.get_command_handler(command)
        if not handler or not callable(handler):
            raise Problem(_("Error! Invalid command `%s`.") % escape(command))
        kwargs.pop("csrfmiddlewaretoken", None)  # The CSRF token should never be passed as a kwarg
        kwargs.pop("command", None)  # Nor the command
        kwargs.update(request=self.request, basket=self.basket)
        kwargs = self.preprocess_kwargs(command, kwargs)

        response = handler(**kwargs) or {}

    except (Problem, ValidationError) as exc:
        if not self.ajax:
            raise
        msg = exc.message if hasattr(exc, "message") else exc
        response = {
            "error": force_text(msg, errors="ignore"),
            "code": force_text(getattr(exc, "code", None) or "", errors="ignore"),
        }

    response = self.postprocess_response(command, kwargs, response)

    if self.ajax:
        return JsonResponse(response)

    return_url = response.get("return") or kwargs.get("return")
    if return_url and return_url.startswith("/"):
        return HttpResponseRedirect(return_url)
    return redirect("shuup:basket")
//  [python]
def handle(self, command, kwargs=None):
    """
    Dispatch and handle processing of the given command.

    :param command: Name of command to run.
    :type command: unicode
    :param kwargs: Arguments to pass to the command handler. If empty, `request.POST` is used.
    :type kwargs: dict
    :return: response.
    :rtype: HttpResponse
    """

    kwargs = kwargs or dict(six.iteritems(self.request.POST))
    try:
        handler = self.get_command_handler(command)
        if not handler or not callable(handler):
            raise Problem(_("Error! Invalid command `%s`.") % command)
        kwargs.pop("csrfmiddlewaretoken", None)  # The CSRF token should never be passed as a kwarg
        kwargs.pop("command", None)  # Nor the command
        kwargs.update(request=self.request, basket=self.basket)
        kwargs = self.preprocess_kwargs(command, kwargs)

        response = handler(**kwargs) or {}

    except (Problem, ValidationError) as exc:
        if not self.ajax:
            raise
        msg = exc.message if hasattr(exc, "message") else exc
        response = {
            "error": force_text(msg, errors="ignore"),
            "code": force_text(getattr(exc, "code", None) or "", errors="ignore"),
        }

    response = self.postprocess_response(command, kwargs, response)

    if self.ajax:
        return JsonResponse(response)

    return_url = response.get("return") or kwargs.get("return")
    if return_url and return_url.startswith("/"):
        return HttpResponseRedirect(return_url)
    return redirect("shuup:basket")
//  [python]
def execute_cmd(cmd, from_async=False):
	"""execute a request as python module"""
	for hook in frappe.get_hooks("override_whitelisted_methods", {}).get(cmd, []):
		# override using the first hook
		cmd = hook
		break

	# via server script
	if run_server_script_api(cmd):
		return None

	try:
		method = get_attr(cmd)
	except Exception as e:
		if frappe.local.conf.developer_mode:
			raise e
		else:
			frappe.respond_as_web_page(title='Invalid Method', html='Method not found',
			indicator_color='red', http_status_code=404)
		return

	if from_async:
		method = method.queue

	if method != run_doc_method:
		is_whitelisted(method)
		is_valid_http_method(method)

	return frappe.call(method, **frappe.form_dict)
//  [python]
def execute_cmd(cmd, from_async=False):
	"""execute a request as python module"""
	for hook in frappe.get_hooks("override_whitelisted_methods", {}).get(cmd, []):
		# override using the first hook
		cmd = hook
		break

	# via server script
	if run_server_script_api(cmd):
		return None

	try:
		method = get_attr(cmd)
	except Exception as e:
		if frappe.local.conf.developer_mode:
			raise e
		else:
			frappe.respond_as_web_page(title='Invalid Method', html='Method not found',
			indicator_color='red', http_status_code=404)
		return

	if from_async:
		method = method.queue

	is_whitelisted(method)
	is_valid_http_method(method)

	return frappe.call(method, **frappe.form_dict)
//  [python]
def testStringNGramsBadDataSplits(self, splits):
  data = ["aa", "bb", "cc", "dd", "ee", "ff"]
  with self.assertRaisesRegex(
      errors.InvalidArgumentError,
      r"Invalid split value|First split value must be 0"):
    self.evaluate(
        gen_string_ops.string_n_grams(
            data=data,
            data_splits=splits,
            separator="",
            ngram_widths=[2],
            left_pad="",
            right_pad="",
            pad_width=0,
            preserve_short_sequences=False))
//  [python]
def testStringNGramsBadDataSplits(self, splits):
  data = ["aa", "bb", "cc", "dd", "ee", "ff"]
  with self.assertRaisesRegex(errors.InvalidArgumentError,
                              "Invalid split value"):
    self.evaluate(
        gen_string_ops.string_n_grams(
            data=data,
            data_splits=splits,
            separator="",
            ngram_widths=[2],
            left_pad="",
            right_pad="",
            pad_width=0,
            preserve_short_sequences=False))
//  [python]
def gravatar(context, user, size=None):
    """
    Outputs the HTML for displaying a user's gravatar.

    This can take an optional size of the image (defaults to 80 if not
    specified).

    This is also influenced by the following settings:

        GRAVATAR_SIZE    - Default size for gravatars
        GRAVATAR_RATING  - Maximum allowed rating (g, pg, r, x)
        GRAVATAR_DEFAULT - Default image set to show if the user hasn't
                           specified a gravatar (identicon, monsterid, wavatar)

    See http://www.gravatar.com/ for more information.
    """
    url = get_gravatar_url(context['request'], user, size)

    if url:
        return format_html(
            '<img src="{0}" width="{1}" height="{1}" alt="{2}" '
            'class="gravatar"/>',
            url, size, user.get_full_name() or user.username)
    else:
        return ''
//  [python]
def gravatar(context, user, size=None):
    """
    Outputs the HTML for displaying a user's gravatar.

    This can take an optional size of the image (defaults to 80 if not
    specified).

    This is also influenced by the following settings:

        GRAVATAR_SIZE    - Default size for gravatars
        GRAVATAR_RATING  - Maximum allowed rating (g, pg, r, x)
        GRAVATAR_DEFAULT - Default image set to show if the user hasn't
                           specified a gravatar (identicon, monsterid, wavatar)

    See http://www.gravatar.com/ for more information.
    """
    url = get_gravatar_url(context['request'], user, size)

    if url:
        return ('<img src="%s" width="%s" height="%s" alt="%s" '
                '     class="gravatar"/>' %
                (url, size, size, user.get_full_name() or user.username))
    else:
        return ''
//  [python]
def scriptPath(*pathSegs):
    startPath = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
    path = os.path.join(startPath, *pathSegs)
    return path
//  [python]
def scriptPath(*pathSegs):
    startPath = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
    return os.path.join(startPath, *pathSegs)
//  [python]
def _redirect_request_using_get(self, request, redirect_url):
    redirect_request = _build_redirect_request(
        request,
        url=redirect_url,
        method='GET',
        body='',
    )
    redirect_request.headers.pop('Content-Type', None)
    redirect_request.headers.pop('Content-Length', None)
    return redirect_request
//  [python]
def _redirect_request_using_get(self, request, redirect_url):
    redirected = request.replace(url=redirect_url, method='GET', body='')
    redirected.headers.pop('Content-Type', None)
    redirected.headers.pop('Content-Length', None)
    return redirected
//  [python]
def is_safe_url(url, host=None):
    """
    Return ``True`` if the url is a safe redirection (i.e. it doesn't point to
    a different host and uses a safe scheme).

    Always returns ``False`` on an empty url.
    """
    if not url:
        return False
    url_info = urllib_parse.urlparse(url)
    return (not url_info.netloc or url_info.netloc == host) and \
        (not url_info.scheme or url_info.scheme in ['http', 'https'])
//  [python]
def is_safe_url(url, host=None):
    """
    Return ``True`` if the url is a safe redirection (i.e. it doesn't point to
    a different host).

    Always returns ``False`` on an empty url.
    """
    if not url:
        return False
    netloc = urllib_parse.urlparse(url)[1]
    return not netloc or netloc == host
//  [python]
def test_file(self, tmpdir):
    filename = tmpdir / 'foo'
    filename.ensure()
    url = QUrl.fromLocalFile(str(filename))
    req = QNetworkRequest(url)
    reply = filescheme.handler(req, None, None)
    assert reply is None
//  [python]
def test_file(self, tmpdir):
    filename = tmpdir / 'foo'
    filename.ensure()
    url = QUrl.fromLocalFile(str(filename))
    req = QNetworkRequest(url)
    reply = filescheme.handler(req)
    assert reply is None
//  [python]
def test_dir(self, tmpdir):
    url = QUrl.fromLocalFile(str(tmpdir))
    req = QNetworkRequest(url)
    reply = filescheme.handler(req, None, None)
    # The URL will always use /, even on Windows - so we force this here
    # too.
    tmpdir_path = str(tmpdir).replace(os.sep, '/')
    assert reply.readAll() == filescheme.dirbrowser_html(tmpdir_path)
//  [python]
def test_dir(self, tmpdir):
    url = QUrl.fromLocalFile(str(tmpdir))
    req = QNetworkRequest(url)
    reply = filescheme.handler(req)
    # The URL will always use /, even on Windows - so we force this here
    # too.
    tmpdir_path = str(tmpdir).replace(os.sep, '/')
    assert reply.readAll() == filescheme.dirbrowser_html(tmpdir_path)
//  [python]
def process_response(self, request, response, spider):
    if (
        request.meta.get('dont_redirect', False)
        or response.status in getattr(spider, 'handle_httpstatus_list', [])
        or response.status in request.meta.get('handle_httpstatus_list', [])
        or request.meta.get('handle_httpstatus_all', False)
    ):
        return response

    allowed_status = (301, 302, 303, 307, 308)
    if 'Location' not in response.headers or response.status not in allowed_status:
        return response

    location = safe_url_string(response.headers['Location'])
    if response.headers['Location'].startswith(b'//'):
        request_scheme = urlparse(request.url).scheme
        location = request_scheme + '://' + location.lstrip('/')

    redirected_url = urljoin(request.url, location)

    if response.status in (301, 307, 308) or request.method == 'HEAD':
        redirected = _build_redirect_request(request, url=redirected_url)
        return self._redirect(redirected, request, spider, response.status)

    redirected = self._redirect_request_using_get(request, redirected_url)
    return self._redirect(redirected, request, spider, response.status)
//  [python]
def process_response(self, request, response, spider):
    if (
        request.meta.get('dont_redirect', False)
        or response.status in getattr(spider, 'handle_httpstatus_list', [])
        or response.status in request.meta.get('handle_httpstatus_list', [])
        or request.meta.get('handle_httpstatus_all', False)
    ):
        return response

    allowed_status = (301, 302, 303, 307, 308)
    if 'Location' not in response.headers or response.status not in allowed_status:
        return response

    location = safe_url_string(response.headers['Location'])
    if response.headers['Location'].startswith(b'//'):
        request_scheme = urlparse(request.url).scheme
        location = request_scheme + '://' + location.lstrip('/')

    redirected_url = urljoin(request.url, location)

    if response.status in (301, 307, 308) or request.method == 'HEAD':
        redirected = request.replace(url=redirected_url)
        return self._redirect(redirected, request, spider, response.status)

    redirected = self._redirect_request_using_get(request, redirected_url)
    return self._redirect(redirected, request, spider, response.status)
//  [python]
def test_open_with_filename(self):
    with NamedTemporaryFile() as tmp:
        fp = memmap(tmp.name, dtype=self.dtype, mode='w+',
                    shape=self.shape)
        fp[:] = self.data[:]
        del fp
//  [python]
def test_open_with_filename(self):
    tmpname = mktemp('', 'mmap')
    fp = memmap(tmpname, dtype=self.dtype, mode='w+',
                   shape=self.shape)
    fp[:] = self.data[:]
    del fp
    os.unlink(tmpname)
//  [python]
def edit_book_comments(comments, book):
    modif_date = False
    if comments:
        comments = clean_html(comments)
    if len(book.comments):
        if book.comments[0].text != comments:
            book.comments[0].text = clean_html(comments)
            modif_date = True
    else:
        if comments:
            book.comments.append(db.Comments(text=comments, book=book.id))
            modif_date = True
    return modif_date
//  [python]
def edit_book_comments(comments, book):
    modif_date = False
    if len(book.comments):
        if book.comments[0].text != comments:
            book.comments[0].text = comments
            modif_date = True
    else:
        if comments:
            book.comments.append(db.Comments(text=comments, book=book.id))
            modif_date = True
    return modif_date
//  [python]
def setup_logging():
    """Configure the python logging appropriately for the tests.

    (Logs will end up in _trial_temp.)
    """
    root_logger = logging.getLogger()

    log_format = "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s" " - %(message)s"

    handler = ToTwistedHandler()
    formatter = logging.Formatter(log_format)
    handler.setFormatter(formatter)
    root_logger.addHandler(handler)

    log_level = os.environ.get("SYDENT_TEST_LOG_LEVEL", "ERROR")
    root_logger.setLevel(log_level)
//  [python]
def setup_logging():
    """Configure the python logging appropriately for the tests.

    (Logs will end up in _trial_temp.)
    """
    root_logger = logging.getLogger()

    log_format = (
        "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s"
        " - %(message)s"
    )

    handler = ToTwistedHandler()
    formatter = logging.Formatter(log_format)
    handler.setFormatter(formatter)
    root_logger.addHandler(handler)

    log_level = os.environ.get("SYDENT_TEST_LOG_LEVEL", "ERROR")
    root_logger.setLevel(log_level)
//  [python]
def test_unicode_encode_error(self, mocker):
    url = QUrl('file:///tmp/foo')
    req = QNetworkRequest(url)

    err = UnicodeEncodeError('ascii', '', 0, 2, 'foo')
    mocker.patch('os.path.isdir', side_effect=err)

    reply = filescheme.handler(req, None, None)
    assert reply is None
//  [python]
def test_unicode_encode_error(self, mocker):
    url = QUrl('file:///tmp/foo')
    req = QNetworkRequest(url)

    err = UnicodeEncodeError('ascii', '', 0, 2, 'foo')
    mocker.patch('os.path.isdir', side_effect=err)

    reply = filescheme.handler(req)
    assert reply is None
//  [python]
def make_sydent(test_config={}):
    """Create a new sydent

    Args:
        test_config (dict): any configuration variables for overriding the default sydent
            config
    """
    # Use an in-memory SQLite database. Note that the database isn't cleaned up between
    # tests, so by default the same database will be used for each test if changed to be
    # a file on disk.
    if "db" not in test_config:
        test_config["db"] = {"db.file": ":memory:"}
    else:
        test_config["db"].setdefault("db.file", ":memory:")

    reactor = ResolvingMemoryReactorClock()
    return Sydent(reactor=reactor, cfg=parse_config_dict(test_config), use_tls_for_federation=False)
//  [python]
def make_sydent(test_config={}):
    """Create a new sydent

    Args:
        test_config (dict): any configuration variables for overriding the default sydent
            config
    """
    # Use an in-memory SQLite database. Note that the database isn't cleaned up between
    # tests, so by default the same database will be used for each test if changed to be
    # a file on disk.
    if 'db' not in test_config:
        test_config['db'] = {'db.file': ':memory:'}
    else:
        test_config['db'].setdefault('db.file', ':memory:')

    reactor = MemoryReactorClock()
    return Sydent(reactor=reactor, cfg=parse_config_dict(test_config))
//  [python]
def is_valid_hostname(string: str) -> bool:
    """Validate that a given string is a valid hostname or domain name.

    For domain names, this only validates that the form is right (for
    instance, it doesn't check that the TLD is valid).

    :param string: The string to validate
    :type string: str

    :return: Whether the input is a valid hostname
    :rtype: bool
    """

    return hostname_regex.match(string) is not None
//  [python]
def is_valid_hostname(string: str) -> bool:
    """Validate that a given string is a valid hostname or domain name, with an
    optional port number.

    For domain names, this only validates that the form is right (for
    instance, it doesn't check that the TLD is valid). If a port is
    specified, it has to be a valid port number.

    :param string: The string to validate
    :type string: str

    :return: Whether the input is a valid hostname
    :rtype: bool
    """

    host_parts = string.split(":", 1)

    if len(host_parts) == 1:
        return hostname_regex.match(string) is not None
    else:
        host, port = host_parts
        valid_hostname = hostname_regex.match(host) is not None

        try:
            port_num = int(port)
            valid_port = (
                port == str(port_num)  # exclude things like '08090' or ' 8090'
                and 1 <= port_num < 65536)
        except ValueError:
            valid_port = False

        return valid_hostname and valid_port
//  [python]
def _not_here_yet(request, *args, **kwargs):
    return HttpResponse("Not here yet: %s (%r, %r)" % (request.path, escape(args), escape(kwargs)), status=410)
//  [python]
def _not_here_yet(request, *args, **kwargs):
    return HttpResponse("Not here yet: %s (%r, %r)" % (request.path, args, kwargs), status=410)
//  [python]
def load_hparams_from_yaml(config_yaml: str, use_omegaconf: bool = True) -> Dict[str, Any]:
    """Load hparams from a file.

        Args:
            config_yaml: Path to config yaml file
            use_omegaconf: If omegaconf is available and ``use_omegaconf=True``,
                the hparams will be converted to ``DictConfig`` if possible.

    >>> hparams = Namespace(batch_size=32, learning_rate=0.001, data_root='./any/path/here')
    >>> path_yaml = './testing-hparams.yaml'
    >>> save_hparams_to_yaml(path_yaml, hparams)
    >>> hparams_new = load_hparams_from_yaml(path_yaml)
    >>> vars(hparams) == hparams_new
    True
    >>> os.remove(path_yaml)
    """
    fs = get_filesystem(config_yaml)
    if not fs.exists(config_yaml):
        rank_zero_warn(f"Missing Tags: {config_yaml}.", category=RuntimeWarning)
        return {}

    with fs.open(config_yaml, "r") as fp:
        hparams = yaml.full_load(fp)

    if _OMEGACONF_AVAILABLE:
        if use_omegaconf:
            try:
                return OmegaConf.create(hparams)
            except (UnsupportedValueType, ValidationError):
                pass
    return hparams
//  [python]
def load_hparams_from_yaml(config_yaml: str, use_omegaconf: bool = True) -> Dict[str, Any]:
    """Load hparams from a file.

        Args:
            config_yaml: Path to config yaml file
            use_omegaconf: If omegaconf is available and ``use_omegaconf=True``,
                the hparams will be converted to ``DictConfig`` if possible.

    >>> hparams = Namespace(batch_size=32, learning_rate=0.001, data_root='./any/path/here')
    >>> path_yaml = './testing-hparams.yaml'
    >>> save_hparams_to_yaml(path_yaml, hparams)
    >>> hparams_new = load_hparams_from_yaml(path_yaml)
    >>> vars(hparams) == hparams_new
    True
    >>> os.remove(path_yaml)
    """
    fs = get_filesystem(config_yaml)
    if not fs.exists(config_yaml):
        rank_zero_warn(f"Missing Tags: {config_yaml}.", category=RuntimeWarning)
        return {}

    with fs.open(config_yaml, "r") as fp:
        hparams = yaml.load(fp, Loader=yaml.UnsafeLoader)

    if _OMEGACONF_AVAILABLE:
        if use_omegaconf:
            try:
                return OmegaConf.create(hparams)
            except (UnsupportedValueType, ValidationError):
                pass
    return hparams
//  [python]
def bm_to_item(self, bm):
    return bm.copy()
//  [python]
def bm_to_item(self, bm):
    return bytearray(cPickle.dumps(bm, -1))
//  [python]
def export_bookmarks(self):
    filename = choose_save_file(
        self, 'export-viewer-bookmarks', _('Export bookmarks'),
        filters=[(_('Saved bookmarks'), ['calibre-bookmarks'])], all_files=False, initial_filename='bookmarks.calibre-bookmarks')
    if filename:
        with lopen(filename, 'wb') as fileobj:
            fileobj.write(json.dumps(self.get_bookmarks(), indent=True))
//  [python]
def export_bookmarks(self):
    filename = choose_save_file(
        self, 'export-viewer-bookmarks', _('Export bookmarks'),
        filters=[(_('Saved bookmarks'), ['pickle'])], all_files=False, initial_filename='bookmarks.pickle')
    if filename:
        with open(filename, 'wb') as fileobj:
            cPickle.dump(self.get_bookmarks(), fileobj, -1)
//  [python]
def import_bookmarks(self):
    files = choose_files(self, 'export-viewer-bookmarks', _('Import bookmarks'),
        filters=[(_('Saved bookmarks'), ['calibre-bookmarks'])], all_files=False, select_only_single_file=True)
    if not files:
        return
    filename = files[0]

    imported = None
    with lopen(filename, 'rb') as fileobj:
        imported = json.load(fileobj)

    if imported is not None:
        bad = False
        try:
            for bm in imported:
                if 'title' not in bm:
                    bad = True
                    break
        except Exception:
            pass

        if not bad:
            bookmarks = self.get_bookmarks()
            for bm in imported:
                if bm not in bookmarks:
                    bookmarks.append(bm)
            self.set_bookmarks([bm for bm in bookmarks if bm['title'] != 'calibre_current_page_bookmark'])
            self.edited.emit(self.get_bookmarks())
//  [python]
def import_bookmarks(self):
    files = choose_files(self, 'export-viewer-bookmarks', _('Import bookmarks'),
        filters=[(_('Saved bookmarks'), ['pickle'])], all_files=False, select_only_single_file=True)
    if not files:
        return
    filename = files[0]

    imported = None
    with open(filename, 'rb') as fileobj:
        imported = cPickle.load(fileobj)

    if imported is not None:
        bad = False
        try:
            for bm in imported:
                if 'title' not in bm:
                    bad = True
                    break
        except Exception:
            pass

        if not bad:
            bookmarks = self.get_bookmarks()
            for bm in imported:
                if bm not in bookmarks:
                    bookmarks.append(bm)
            self.set_bookmarks([bm for bm in bookmarks if bm['title'] != 'calibre_current_page_bookmark'])
            self.edited.emit(self.get_bookmarks())
//  [python]
def __init__(self, **kwargs):
    """Initialize HTMLSerializer.

    Keyword options (default given first unless specified) include:

    inject_meta_charset=True|False
      Whether it insert a meta element to define the character set of the
      document.
    quote_attr_values="legacy"|"spec"|"always"
      Whether to quote attribute values that don't require quoting
      per legacy browser behaviour, when required by the standard, or always.
    quote_char=u'"'|u"'"
      Use given quote character for attribute quoting. Default is to
      use double quote unless attribute value contains a double quote,
      in which case single quotes are used instead.
    escape_lt_in_attrs=False|True
      Whether to escape < in attribute values.
    escape_rcdata=False|True
      Whether to escape characters that need to be escaped within normal
      elements within rcdata elements such as style.
    resolve_entities=True|False
      Whether to resolve named character entities that appear in the
      source tree. The XML predefined entities &lt; &gt; &amp; &quot; &apos;
      are unaffected by this setting.
    strip_whitespace=False|True
      Whether to remove semantically meaningless whitespace. (This
      compresses all whitespace to a single space except within pre.)
    minimize_boolean_attributes=True|False
      Shortens boolean attributes to give just the attribute value,
      for example <input disabled="disabled"> becomes <input disabled>.
    use_trailing_solidus=False|True
      Includes a close-tag slash at the end of the start tag of void
      elements (empty elements whose end tag is forbidden). E.g. <hr/>.
    space_before_trailing_solidus=True|False
      Places a space immediately before the closing slash in a tag
      using a trailing solidus. E.g. <hr />. Requires use_trailing_solidus.
    sanitize=False|True
      Strip all unsafe or unknown constructs from output.
      See `html5lib user documentation`_
    omit_optional_tags=True|False
      Omit start/end tags that are optional.
    alphabetical_attributes=False|True
      Reorder attributes to be in alphabetical order.

    .. _html5lib user documentation: http://code.google.com/p/html5lib/wiki/UserDocumentation
    """
    if 'quote_char' in kwargs:
        self.use_best_quote_char = False
    for attr in self.options:
        setattr(self, attr, kwargs.get(attr, getattr(self, attr)))
    self.errors = []
    self.strict = False
//  [python]
def __init__(self, **kwargs):
    """Initialize HTMLSerializer.

    Keyword options (default given first unless specified) include:

    inject_meta_charset=True|False
      Whether it insert a meta element to define the character set of the
      document.
    quote_attr_values=True|False
      Whether to quote attribute values that don't require quoting
      per HTML5 parsing rules.
    quote_char=u'"'|u"'"
      Use given quote character for attribute quoting. Default is to
      use double quote unless attribute value contains a double quote,
      in which case single quotes are used instead.
    escape_lt_in_attrs=False|True
      Whether to escape < in attribute values.
    escape_rcdata=False|True
      Whether to escape characters that need to be escaped within normal
      elements within rcdata elements such as style.
    resolve_entities=True|False
      Whether to resolve named character entities that appear in the
      source tree. The XML predefined entities &lt; &gt; &amp; &quot; &apos;
      are unaffected by this setting.
    strip_whitespace=False|True
      Whether to remove semantically meaningless whitespace. (This
      compresses all whitespace to a single space except within pre.)
    minimize_boolean_attributes=True|False
      Shortens boolean attributes to give just the attribute value,
      for example <input disabled="disabled"> becomes <input disabled>.
    use_trailing_solidus=False|True
      Includes a close-tag slash at the end of the start tag of void
      elements (empty elements whose end tag is forbidden). E.g. <hr/>.
    space_before_trailing_solidus=True|False
      Places a space immediately before the closing slash in a tag
      using a trailing solidus. E.g. <hr />. Requires use_trailing_solidus.
    sanitize=False|True
      Strip all unsafe or unknown constructs from output.
      See `html5lib user documentation`_
    omit_optional_tags=True|False
      Omit start/end tags that are optional.
    alphabetical_attributes=False|True
      Reorder attributes to be in alphabetical order.

    .. _html5lib user documentation: http://code.google.com/p/html5lib/wiki/UserDocumentation
    """
    if 'quote_char' in kwargs:
        self.use_best_quote_char = False
    for attr in self.options:
        setattr(self, attr, kwargs.get(attr, getattr(self, attr)))
    self.errors = []
    self.strict = False
//  [python]
def get_epub_info(tmp_file_path, original_file_name, original_file_extension):
    ns = {
        'n': 'urn:oasis:names:tc:opendocument:xmlns:container',
        'pkg': 'http://www.idpf.org/2007/opf',
        'dc': 'http://purl.org/dc/elements/1.1/'
    }

    epub_zip = zipfile.ZipFile(tmp_file_path)

    txt = epub_zip.read('META-INF/container.xml')
    tree = etree.fromstring(txt)
    cf_name = tree.xpath('n:rootfiles/n:rootfile/@full-path', namespaces=ns)[0]
    cf = epub_zip.read(cf_name)
    tree = etree.fromstring(cf)

    cover_path = os.path.dirname(cf_name)

    p = tree.xpath('/pkg:package/pkg:metadata', namespaces=ns)[0]

    epub_metadata = {}

    for s in ['title', 'description', 'creator', 'language', 'subject']:
        tmp = p.xpath('dc:%s/text()' % s, namespaces=ns)
        if len(tmp) > 0:
            if s == 'creator':
                epub_metadata[s] = ' & '.join(split_authors(tmp))
            elif s == 'subject':
                epub_metadata[s] = ', '.join(tmp)
            else:
                epub_metadata[s] = tmp[0]
        else:
            epub_metadata[s] = 'Unknown'

    if epub_metadata['subject'] == 'Unknown':
        epub_metadata['subject'] = ''

    if epub_metadata['description'] == u'Unknown':
        description = tree.xpath("//*[local-name() = 'description']/text()")
        if len(description) > 0:
            epub_metadata['description'] = description
        else:
            epub_metadata['description'] = ""

    lang = epub_metadata['language'].split('-', 1)[0].lower()
    epub_metadata['language'] = isoLanguages.get_lang3(lang)

    epub_metadata = parse_epub_series(ns, tree, epub_metadata)

    cover_file = parse_epub_cover(ns, tree, epub_zip, cover_path, tmp_file_path)

    if not epub_metadata['title']:
        title = original_file_name
    else:
        title = epub_metadata['title']

    return BookMeta(
        file_path=tmp_file_path,
        extension=original_file_extension,
        title=title.encode('utf-8').decode('utf-8'),
        author=epub_metadata['creator'].encode('utf-8').decode('utf-8'),
        cover=cover_file,
        description=epub_metadata['description'],
        tags=epub_metadata['subject'].encode('utf-8').decode('utf-8'),
        series=epub_metadata['series'].encode('utf-8').decode('utf-8'),
        series_id=epub_metadata['series_id'].encode('utf-8').decode('utf-8'),
        languages=epub_metadata['language'],
        publisher="")
//  [python]
def get_epub_info(tmp_file_path, original_file_name, original_file_extension):
    ns = {
        'n': 'urn:oasis:names:tc:opendocument:xmlns:container',
        'pkg': 'http://www.idpf.org/2007/opf',
        'dc': 'http://purl.org/dc/elements/1.1/'
    }

    epub_zip = zipfile.ZipFile(tmp_file_path)

    txt = epub_zip.read('META-INF/container.xml')
    tree = etree.fromstring(txt)
    cfname = tree.xpath('n:rootfiles/n:rootfile/@full-path', namespaces=ns)[0]
    cf = epub_zip.read(cfname)
    tree = etree.fromstring(cf)

    coverpath = os.path.dirname(cfname)

    p = tree.xpath('/pkg:package/pkg:metadata', namespaces=ns)[0]

    epub_metadata = {}

    for s in ['title', 'description', 'creator', 'language', 'subject']:
        tmp = p.xpath('dc:%s/text()' % s, namespaces=ns)
        if len(tmp) > 0:
            if s == 'creator':
                epub_metadata[s] = ' & '.join(split_authors(tmp))
            elif s == 'subject':
                epub_metadata[s] = ', '.join(tmp)
            else:
                epub_metadata[s] = tmp[0]
        else:
            epub_metadata[s] = 'Unknown'

    if epub_metadata['subject'] == 'Unknown':
        epub_metadata['subject'] = ''

    if epub_metadata['description'] == u'Unknown':
        description = tree.xpath("//*[local-name() = 'description']/text()")
        if len(description) > 0:
            epub_metadata['description'] = description
        else:
            epub_metadata['description'] = ""

    lang = epub_metadata['language'].split('-', 1)[0].lower()
    epub_metadata['language'] = isoLanguages.get_lang3(lang)

    epub_metadata = parse_epub_series(ns, tree, epub_metadata)

    cover_file = parse_epub_cover(ns, tree, epub_zip, coverpath, tmp_file_path)

    if not epub_metadata['title']:
        title = original_file_name
    else:
        title = epub_metadata['title']

    return BookMeta(
        file_path=tmp_file_path,
        extension=original_file_extension,
        title=title.encode('utf-8').decode('utf-8'),
        author=epub_metadata['creator'].encode('utf-8').decode('utf-8'),
        cover=cover_file,
        description=epub_metadata['description'],
        tags=epub_metadata['subject'].encode('utf-8').decode('utf-8'),
        series=epub_metadata['series'].encode('utf-8').decode('utf-8'),
        series_id=epub_metadata['series_id'].encode('utf-8').decode('utf-8'),
        languages=epub_metadata['language'],
        publisher="")
//  [python]
def item_to_bm(self, item):
    return item.data(Qt.UserRole).copy()
//  [python]
def item_to_bm(self, item):
    return cPickle.loads(bytes(item.data(Qt.UserRole)))
//  [python]
def parse_epub_cover(ns, tree, epub_zip, cover_path, tmp_file_path):
    cover_section = tree.xpath("/pkg:package/pkg:manifest/pkg:item[@id='cover-image']/@href", namespaces=ns)
    cover_file = None
    # if len(cover_section) > 0:
    for cs in cover_section:
        cover_file = _extract_cover(epub_zip, cs, cover_path, tmp_file_path)
        if cover_file:
            break
    if not cover_file:
        meta_cover = tree.xpath("/pkg:package/pkg:metadata/pkg:meta[@name='cover']/@content", namespaces=ns)
        if len(meta_cover) > 0:
            cover_section = tree.xpath(
                "/pkg:package/pkg:manifest/pkg:item[@id='"+meta_cover[0]+"']/@href", namespaces=ns)
            if not cover_section:
                cover_section = tree.xpath(
                    "/pkg:package/pkg:manifest/pkg:item[@properties='" + meta_cover[0] + "']/@href", namespaces=ns)
        else:
            cover_section = tree.xpath("/pkg:package/pkg:guide/pkg:reference/@href", namespaces=ns)
        for cs in cover_section:
            filetype = cs.rsplit('.', 1)[-1]
            if filetype == "xhtml" or filetype == "html":  # if cover is (x)html format
                markup = epub_zip.read(os.path.join(cover_path, cs))
                markup_tree = etree.fromstring(markup)
                # no matter xhtml or html with no namespace
                img_src = markup_tree.xpath("//*[local-name() = 'img']/@src")
                # Alternative image source
                if not len(img_src):
                    img_src = markup_tree.xpath("//attribute::*[contains(local-name(), 'href')]")
                if len(img_src):
                    # img_src maybe start with "../"" so fullpath join then relpath to cwd
                    filename = os.path.relpath(os.path.join(os.path.dirname(os.path.join(cover_path, cover_section[0])),
                                                            img_src[0]))
                    cover_file = _extract_cover(epub_zip, filename, "", tmp_file_path)
            else:
                cover_file = _extract_cover(epub_zip, cs, cover_path, tmp_file_path)
            if cover_file: break
    return cover_file
//  [python]
def parse_epub_cover(ns, tree, epub_zip, cover_path, tmp_file_path):
    cover_section = tree.xpath("/pkg:package/pkg:manifest/pkg:item[@id='cover-image']/@href", namespaces=ns)
    cover_file = None
    if len(cover_section) > 0:
        cover_file = _extract_cover(epub_zip, cover_section[0], cover_path, tmp_file_path)
    else:
        meta_cover = tree.xpath("/pkg:package/pkg:metadata/pkg:meta[@name='cover']/@content", namespaces=ns)
        if len(meta_cover) > 0:
            cover_section = tree.xpath(
                "/pkg:package/pkg:manifest/pkg:item[@id='"+meta_cover[0]+"']/@href", namespaces=ns)
            if not cover_section:
                cover_section = tree.xpath(
                    "/pkg:package/pkg:manifest/pkg:item[@properties='" + meta_cover[0] + "']/@href", namespaces=ns)
        else:
            cover_section = tree.xpath("/pkg:package/pkg:guide/pkg:reference/@href", namespaces=ns)
        for cs in cover_section:
            filetype = cs.rsplit('.', 1)[-1]
            if filetype == "xhtml" or filetype == "html":  # if cover is (x)html format
                markup = epub_zip.read(os.path.join(cover_path, cs))
                markup_tree = etree.fromstring(markup)
                # no matter xhtml or html with no namespace
                img_src = markup_tree.xpath("//*[local-name() = 'img']/@src")
                # Alternative image source
                if not len(img_src):
                    img_src = markup_tree.xpath("//attribute::*[contains(local-name(), 'href')]")
                if len(img_src):
                    # img_src maybe start with "../"" so fullpath join then relpath to cwd
                    filename = os.path.relpath(os.path.join(os.path.dirname(os.path.join(cover_path, cover_section[0])),
                                                            img_src[0]))
                    cover_file = _extract_cover(epub_zip, filename, "", tmp_file_path)
            else:
                cover_file = _extract_cover(epub_zip, cs, cover_path, tmp_file_path)
            if cover_file:
                break
    return cover_file
//  [python]
def image(self, request, pk):
    obj = self.get_object()

    if obj.get_space() != request.space:
        raise PermissionDenied(detail='You do not have the required permission to perform this action', code=403)

    serializer = self.serializer_class(obj, data=request.data, partial=True)

    if serializer.is_valid():
        serializer.save()
        image = None
        filetype = ".jpeg"  # fall-back to .jpeg, even if wrong, at least users will know it's an image and most image viewers can open it correctly anyways

        if 'image' in serializer.validated_data:
            image = obj.image
            filetype = mimetypes.guess_extension(serializer.validated_data['image'].content_type) or filetype
        elif 'image_url' in serializer.validated_data:
            try:
                url = serializer.validated_data['image_url']
                if validators.url(url, public=True):
                    response = requests.get(url)
                    image = File(io.BytesIO(response.content))
                    filetype = mimetypes.guess_extension(response.headers['content-type']) or filetype
            except UnidentifiedImageError as e:
                print(e)
                pass
            except MissingSchema as e:
                print(e)
                pass
            except Exception as e:
                print(e)
                pass

        if image is not None:
            img = handle_image(request, image, filetype)
            obj.image = File(img, name=f'{uuid.uuid4()}_{obj.pk}{filetype}')
            obj.save()
            return Response(serializer.data)

    return Response(serializer.errors, 400)
//  [python]
def image(self, request, pk):
    obj = self.get_object()

    if obj.get_space() != request.space:
        raise PermissionDenied(detail='You do not have the required permission to perform this action', code=403)

    serializer = self.serializer_class(obj, data=request.data, partial=True)

    if serializer.is_valid():
        serializer.save()
        image = None
        filetype = ".jpeg" # fall-back to .jpeg, even if wrong, at least users will know it's an image and most image viewers can open it correctly anyways

        if 'image' in serializer.validated_data:
            image = obj.image
            filetype = mimetypes.guess_extension(serializer.validated_data['image'].content_type) or filetype
        elif 'image_url' in serializer.validated_data:
            try:
                response = requests.get(serializer.validated_data['image_url'])
                image = File(io.BytesIO(response.content))
                filetype = mimetypes.guess_extension(response.headers['content-type']) or filetype
            except UnidentifiedImageError as e:
                print(e)
                pass
            except MissingSchema as e:
                print(e)
                pass
            except Exception as e:
                print(e)
                pass

        if image is not None:
            img = handle_image(request, image, filetype)
            obj.image = File(img, name=f'{uuid.uuid4()}_{obj.pk}{filetype}')
            obj.save()
            return Response(serializer.data)

    return Response(serializer.errors, 400)