jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def get_type_string(data):
    """
    Translates a Python data type into a string format.
    """
    data_type = type(data)
    
    if data_type in (int, long):
        return 'integer'
    elif data_type == float:
        return 'float'
    elif data_type == bool:
        return 'boolean'
    elif data_type in (list, tuple):
        return 'list'
    elif data_type == dict:
        return 'hash'
    elif data is None:
        return 'null'
    elif isinstance(data, basestring):
        return 'string'
//  [python]
def delete_auth_token(user_id):
    # Invalidate any previously generated Kobo Auth token for this user
    ub.session.query(ub.RemoteAuthToken).filter(ub.RemoteAuthToken.user_id == user_id)\
        .filter(ub.RemoteAuthToken.token_type==1).delete()

    return ub.session_commit()
//  [python]
def delete_auth_token(user_id):
    # Invalidate any prevously generated Kobo Auth token for this user.
    ub.session.query(ub.RemoteAuthToken).filter(ub.RemoteAuthToken.user_id == user_id)\
        .filter(ub.RemoteAuthToken.token_type==1).delete()

    return ub.session_commit()
//  [python]
def to_simple(self, data, options):
    """
    For a piece of data, attempts to recognize it and provide a simplified
    form of something complex.

    This brings complex Python data structures down to native types of the
    serialization format(s).
    """
    if isinstance(data, (list, tuple)):
        return [self.to_simple(item, options) for item in data]
    if isinstance(data, dict):
        return dict((key, self.to_simple(val, options)) for (key, val) in data.iteritems())
    elif isinstance(data, Bundle):
        return dict((key, self.to_simple(val, options)) for (key, val) in data.data.iteritems())
    elif hasattr(data, 'dehydrated_type'):
        if getattr(data, 'dehydrated_type', None) == 'related' and data.is_m2m == False:
            if data.full:
                return self.to_simple(data.fk_resource, options)
            else:
                return self.to_simple(data.value, options)
        elif getattr(data, 'dehydrated_type', None) == 'related' and data.is_m2m == True:
            if data.full:
                return [self.to_simple(bundle, options) for bundle in data.m2m_bundles]
            else:
                return [self.to_simple(val, options) for val in data.value]
        else:
            return self.to_simple(data.value, options)
    elif isinstance(data, datetime.datetime):
        return self.format_datetime(data)
    elif isinstance(data, datetime.date):
        return self.format_date(data)
    elif isinstance(data, datetime.time):
        return self.format_time(data)
    elif isinstance(data, bool):
        return data
    elif type(data) in (long, int, float):
        return data
    elif data is None:
        return None
    else:
        return force_unicode(data)
//  [python]
def to_simple(self, data, options):
    """
    For a piece of data, attempts to recognize it and provide a simplified
    form of something complex.
    
    This brings complex Python data structures down to native types of the
    serialization format(s).
    """
    if isinstance(data, (list, tuple)):
        return [self.to_simple(item, options) for item in data]
    if isinstance(data, dict):
        return dict((key, self.to_simple(val, options)) for (key, val) in data.iteritems())
    elif isinstance(data, Bundle):
        return dict((key, self.to_simple(val, options)) for (key, val) in data.data.iteritems())
    elif hasattr(data, 'dehydrated_type'):
        if getattr(data, 'dehydrated_type', None) == 'related' and data.is_m2m == False:
            if data.full:
                return self.to_simple(data.fk_resource, options)
            else:
                return self.to_simple(data.value, options)
        elif getattr(data, 'dehydrated_type', None) == 'related' and data.is_m2m == True:
            if data.full:
                return [self.to_simple(bundle, options) for bundle in data.m2m_bundles]
            else:
                return [self.to_simple(val, options) for val in data.value]
        else:
            return self.to_simple(data.value, options)
    elif isinstance(data, datetime.datetime):
        return self.format_datetime(data)
    elif isinstance(data, datetime.date):
        return self.format_date(data)
    elif isinstance(data, datetime.time):
        return self.format_time(data)
    elif isinstance(data, bool):
        return data
    elif type(data) in (long, int, float):
        return data
    elif data is None:
        return None
    else:
        return force_unicode(data)
//  [python]
def to_yaml(self, data, options=None):
    """
    Given some Python data, produces YAML output.
    """
    options = options or {}

    if yaml is None:
        raise ImproperlyConfigured("Usage of the YAML aspects requires yaml.")

    return yaml.dump(self.to_simple(data, options))
//  [python]
def to_yaml(self, data, options=None):
    """
    Given some Python data, produces YAML output.
    """
    options = options or {}
    
    if yaml is None:
        raise ImproperlyConfigured("Usage of the YAML aspects requires yaml.")
    
    return yaml.dump(self.to_simple(data, options))
//  [python]
def _unpack_returndata(buf, contract_sig, skip_contract_check, context, expr):
    # expr.func._metadata["type"].return_type is more accurate
    # than contract_sig.return_type in the case of JSON interfaces.
    ast_return_t = expr.func._metadata["type"].return_type

    if ast_return_t is None:
        return ["pass"], 0, 0

    # sanity check
    return_t = new_type_to_old_type(ast_return_t)
    check_assign(dummy_node_for_type(return_t), dummy_node_for_type(contract_sig.return_type))

    return_t = calculate_type_for_external_return(return_t)

    abi_return_t = return_t.abi_type

    min_return_size = abi_return_t.min_size()
    max_return_size = abi_return_t.size_bound()
    assert 0 < min_return_size <= max_return_size

    ret_ofst = buf
    ret_len = max_return_size

    # revert when returndatasize is not in bounds
    ret = []
    # runtime: min_return_size <= returndatasize
    if not skip_contract_check:
        ret += [["assert", ["ge", "returndatasize", min_return_size]]]

    encoding = Encoding.ABI

    buf = IRnode.from_list(
        buf,
        typ=return_t,
        location=MEMORY,
        encoding=encoding,
        annotation=f"{expr.node_source_code} returndata buffer",
    )

    assert isinstance(return_t, TupleType)
    # unpack strictly
    if needs_clamp(return_t, encoding):
        buf2 = IRnode.from_list(
            context.new_internal_variable(return_t), typ=return_t, location=MEMORY
        )

        ret.append(make_setter(buf2, buf))
        ret.append(buf2)
    else:
        ret.append(buf)

    return ret, ret_ofst, ret_len
//  [python]
def _unpack_returndata(buf, contract_sig, skip_contract_check, context):
    return_t = contract_sig.return_type
    if return_t is None:
        return ["pass"], 0, 0

    return_t = calculate_type_for_external_return(return_t)
    # if the abi signature has a different type than
    # the vyper type, we need to wrap and unwrap the type
    # so that the ABI decoding works correctly
    should_unwrap_abi_tuple = return_t != contract_sig.return_type

    abi_return_t = return_t.abi_type

    min_return_size = abi_return_t.min_size()
    max_return_size = abi_return_t.size_bound()
    assert 0 < min_return_size <= max_return_size

    ret_ofst = buf
    ret_len = max_return_size

    # revert when returndatasize is not in bounds
    ret = []
    # runtime: min_return_size <= returndatasize
    # TODO move the -1 optimization to IR optimizer
    if not skip_contract_check:
        ret += [["assert", ["gt", "returndatasize", min_return_size - 1]]]

    # add as the last IRnode a pointer to the return data structure

    # the return type has been wrapped by the calling contract;
    # unwrap it so downstream code isn't confused.
    # basically this expands to buf+32 if the return type has been wrapped
    # in a tuple AND its ABI type is dynamic.
    # in most cases, this simply will evaluate to ret.
    # in the special case where the return type has been wrapped
    # in a tuple AND its ABI type is dynamic, it expands to buf+32.
    buf = IRnode(buf, typ=return_t, encoding=_returndata_encoding(contract_sig), location=MEMORY)

    if should_unwrap_abi_tuple:
        buf = get_element_ptr(buf, 0, array_bounds_check=False)

    ret += [buf]

    return ret, ret_ofst, ret_len
//  [python]
def to_xml(self, data, options=None):
    """
    Given some Python data, produces XML output.
    """
    options = options or {}

    if lxml is None:
        raise ImproperlyConfigured("Usage of the XML aspects requires lxml.")

    return tostring(self.to_etree(data, options), xml_declaration=True, encoding='utf-8')
//  [python]
def to_xml(self, data, options=None):
    """
    Given some Python data, produces XML output.
    """
    options = options or {}
    
    if lxml is None:
        raise ImproperlyConfigured("Usage of the XML aspects requires lxml.")
    
    return tostring(self.to_etree(data, options), xml_declaration=True, encoding='utf-8')
//  [python]
def parse_line(s):
    s = s.rstrip()

    r = re.sub(REG_LINE_GPERF, '', s)
    if r != s: return r
    r = re.sub(REG_HASH_FUNC, 'hash(OnigCodePoint codes[])', s)
    if r != s: return r
    r = re.sub(REG_STR_AT, 'onig_codes_byte_at(codes, \\1)', s)
    if r != s: return r
    r = re.sub(REG_UNFOLD_KEY, 'unicode_unfold_key(OnigCodePoint code)', s)
    if r != s: return r
    r = re.sub(REG_ENTRY, '{\\1, \\2, \\3}', s)
    if r != s: return r
    r = re.sub(REG_EMPTY_ENTRY, '{0xffffffff, \\1, \\2}', s)
    if r != s: return r
    r = re.sub(REG_IF_LEN, 'if (0 == 0)', s)
    if r != s: return r
    r = re.sub(REG_GET_HASH, 'int key = hash(&code);', s)
    if r != s: return r
    r = re.sub(REG_GET_CODE, 'OnigCodePoint gcode = wordlist[key].code;', s)
    if r != s: return r
    r = re.sub(REG_CODE_CHECK, 'if (code == gcode && wordlist[key].index >= 0)', s)
    if r != s: return r

    return s
//  [python]
def parse_line(s):
    s = s.rstrip()

    r = re.sub(REG_LINE_GPERF, '', s)
    if r != s: return r
    r = re.sub(REG_HASH_FUNC, 'hash(OnigCodePoint codes[])', s)
    if r != s: return r
    r = re.sub(REG_STR_AT, 'onig_codes_byte_at(codes, \\1)', s)
    if r != s: return r
    r = re.sub(REG_UNFOLD_KEY, 'unicode_unfold_key(OnigCodePoint code)', s)
    if r != s: return r
    r = re.sub(REG_ENTRY, '{\\1, \\2, \\3}', s)
    if r != s: return r
    r = re.sub(REG_EMPTY_ENTRY, '{0xffffffff, \\1, \\2}', s)
    if r != s: return r
    r = re.sub(REG_IF_LEN, 'if (0 == 0)', s)
    if r != s: return r
    r = re.sub(REG_GET_HASH, 'int key = hash(&code);', s)
    if r != s: return r
    r = re.sub(REG_GET_CODE, 'OnigCodePoint gcode = wordlist[key].code;', s)
    if r != s: return r
    r = re.sub(REG_CODE_CHECK, 'if (code == gcode)', s)
    if r != s: return r

    return s
//  [python]
def parse_configuration_file(config_path):
    """
    Read the config file for an experiment to get ParlAI settings.

    :param config_path:
        path to config

    :return:
        parsed configuration dictionary
    """
    result = {}
    result["configs"] = {}
    with open(config_path) as f:
        cfg = yaml.load(f.read(), Loader=yaml.SafeLoader)
        # get world path
        result["world_path"] = cfg.get("world_module")
        if not result["world_path"]:
            raise ValueError("Did not specify world module")
        result["overworld"] = cfg.get("overworld")
        if not result["overworld"]:
            raise ValueError("Did not specify overworld")
        result["max_workers"] = cfg.get("max_workers")
        if not result["max_workers"]:
            raise ValueError("Did not specify max_workers")
        result["task_name"] = cfg.get("task_name")
        if not result["task_name"]:
            raise ValueError("Did not specify task name")
        task_world = cfg.get("tasks")
        if task_world is None or len(task_world) == 0:
            raise ValueError("task not in config file")
        # get task file
        for task_name, configuration in task_world.items():
            if "task_world" not in configuration:
                raise ValueError("{} does not specify a task".format(task_name))
            result["configs"][task_name] = WorldConfig(
                world_name=task_name,
                onboarding_name=configuration.get("onboard_world"),
                task_name=configuration.get("task_world"),
                max_time_in_pool=configuration.get("timeout") or 300,
                agents_required=configuration.get("agents_required") or 1,
                backup_task=configuration.get("backup_task"),
            )
        # get world options, additional args
        result["world_opt"] = cfg.get("opt", {})
        result["additional_args"] = cfg.get("additional_args", {})

    return result
//  [python]
def parse_configuration_file(config_path):
    """
    Read the config file for an experiment to get ParlAI settings.

    :param config_path:
        path to config

    :return:
        parsed configuration dictionary
    """
    result = {}
    result["configs"] = {}
    with open(config_path) as f:
        cfg = yaml.load(f.read(), Loader=yaml.FullLoader)
        # get world path
        result["world_path"] = cfg.get("world_module")
        if not result["world_path"]:
            raise ValueError("Did not specify world module")
        result["overworld"] = cfg.get("overworld")
        if not result["overworld"]:
            raise ValueError("Did not specify overworld")
        result["max_workers"] = cfg.get("max_workers")
        if not result["max_workers"]:
            raise ValueError("Did not specify max_workers")
        result["task_name"] = cfg.get("task_name")
        if not result["task_name"]:
            raise ValueError("Did not specify task name")
        task_world = cfg.get("tasks")
        if task_world is None or len(task_world) == 0:
            raise ValueError("task not in config file")
        # get task file
        for task_name, configuration in task_world.items():
            if "task_world" not in configuration:
                raise ValueError("{} does not specify a task".format(task_name))
            result["configs"][task_name] = WorldConfig(
                world_name=task_name,
                onboarding_name=configuration.get("onboard_world"),
                task_name=configuration.get("task_world"),
                max_time_in_pool=configuration.get("timeout") or 300,
                agents_required=configuration.get("agents_required") or 1,
                backup_task=configuration.get("backup_task"),
            )
        # get world options, additional args
        result["world_opt"] = cfg.get("opt", {})
        result["additional_args"] = cfg.get("additional_args", {})

    return result
//  [python]
def generate_auth_token(user_id):
    warning = False
    host_list = request.host.rsplit(':')
    if len(host_list) == 1:
        host = ':'.join(host_list)
    else:
        host = ':'.join(host_list[0:-1])
    if host.startswith('127.') or host.lower() == 'localhost' or host.startswith('[::ffff:7f') or host == "[::1]":
        warning = _('Please access Calibre-Web from non localhost to get valid api_endpoint for kobo device')

    # Generate auth token if none is existing for this user
    auth_token = ub.session.query(ub.RemoteAuthToken).filter(
        ub.RemoteAuthToken.user_id == user_id
    ).filter(ub.RemoteAuthToken.token_type==1).first()

    if not auth_token:
        auth_token = ub.RemoteAuthToken()
        auth_token.user_id = user_id
        auth_token.expiration = datetime.max
        auth_token.auth_token = (hexlify(urandom(16))).decode("utf-8")
        auth_token.token_type = 1

        ub.session.add(auth_token)
        ub.session_commit()

    books = calibre_db.session.query(db.Books).join(db.Data).all()

    for book in books:
        formats = [data.format for data in book.data]
        if not 'KEPUB' in formats and config.config_kepubifypath and 'EPUB' in formats:
            helper.convert_book_format(book.id, config.config_calibre_dir, 'EPUB', 'KEPUB', current_user.name)

    return render_title_template(
        "generate_kobo_auth_url.html",
        title=_(u"Kobo Setup"),
        auth_token=auth_token.auth_token,
        warning = warning
    )
//  [python]
def generate_auth_token(user_id):
    host_list = request.host.rsplit(':')
    if len(host_list) == 1:
        host = ':'.join(host_list)
    else:
        host = ':'.join(host_list[0:-1])
    if host.startswith('127.') or host.lower() == 'localhost' or host.startswith('[::ffff:7f'):
        warning = _('PLease access calibre-web from non localhost to get valid api_endpoint for kobo device')
        return render_title_template(
            "generate_kobo_auth_url.html",
            title=_(u"Kobo Setup"),
            warning = warning
        )
    else:
        # Invalidate any prevously generated Kobo Auth token for this user.
        auth_token = ub.session.query(ub.RemoteAuthToken).filter(
            ub.RemoteAuthToken.user_id == user_id
        ).filter(ub.RemoteAuthToken.token_type==1).first()

        if not auth_token:
            auth_token = ub.RemoteAuthToken()
            auth_token.user_id = user_id
            auth_token.expiration = datetime.max
            auth_token.auth_token = (hexlify(urandom(16))).decode("utf-8")
            auth_token.token_type = 1

            ub.session.add(auth_token)
            ub.session_commit()

        books = calibre_db.session.query(db.Books).join(db.Data).all()

        for book in books:
            formats = [data.format for data in book.data]
            if not 'KEPUB' in formats and config.config_kepubifypath and 'EPUB' in formats:
                helper.convert_book_format(book.id, config.config_calibre_dir, 'EPUB', 'KEPUB', current_user.name)

        return render_title_template(
            "generate_kobo_auth_url.html",
            title=_(u"Kobo Setup"),
            kobo_auth_url=url_for(
                "kobo.TopLevelEndpoint", auth_token=auth_token.auth_token, _external=True
            ),
            warning = False
        )
//  [python]
def _get_index_absolute_path(index):
    return safe_join(INDEXDIR, index)
//  [python]
def _get_index_absolute_path(index):
    return os.path.join(INDEXDIR, index)
//  [python]
def _get_obj_absolute_path(obj_path):
    return safe_join(DATAROOT, obj_path)
//  [python]
def _get_obj_absolute_path(obj_path):
    return os.path.join(DATAROOT, obj_path)
//  [python]
def __call__(self, request):
    file_fields = request.POST.getlist("s3file")
    for field_name in file_fields:

        paths = request.POST.getlist(field_name)
        if paths:
            try:
                signature = request.POST[f"{field_name}-s3f-signature"]
            except KeyError:
                raise PermissionDenied("No signature provided.")
            try:
                request.FILES.setlist(
                    field_name, list(self.get_files_from_storage(paths, signature))
                )
            except SuspiciousFileOperation as e:
                raise PermissionDenied("Illegal file name!") from e

    if local_dev and request.path == "/__s3_mock__/":
        return views.S3MockView.as_view()(request)

    return self.get_response(request)
//  [python]
def __call__(self, request):
    file_fields = request.POST.getlist("s3file")
    for field_name in file_fields:
        paths = request.POST.getlist(field_name)
        request.FILES.setlist(field_name, list(self.get_files_from_storage(paths)))

    if local_dev and request.path == "/__s3_mock__/":
        return views.S3MockView.as_view()(request)

    return self.get_response(request)
//  [python]
def get_files_from_storage(paths, signature):
    """Return S3 file where the name does not include the path."""
    try:
        location = storage.aws_location
    except AttributeError:
        location = storage.location
    signer = signing.Signer(
        salt=f"{S3FileInputMixin.__module__}.{S3FileInputMixin.__name__}"
    )
    for path in paths:
        path = pathlib.PurePosixPath(path)
        print(path)
        print(signer.signature(path.parent), signature)
        if not constant_time_compare(signer.signature(path.parent), signature):
            raise PermissionDenied("Illegal signature!")
        try:
            relative_path = str(path.relative_to(location))
        except ValueError as e:
            raise SuspiciousFileOperation(
                f"Path is not inside the designated upload location: {path}"
            ) from e

        try:
            f = storage.open(relative_path)
            f.name = path.name
            yield f
        except (OSError, ValueError):
            logger.exception("File not found: %s", path)
//  [python]
def get_files_from_storage(paths):
    """Return S3 file where the name does not include the path."""
    for path in paths:
        path = pathlib.PurePosixPath(path)
        try:
            location = storage.aws_location
        except AttributeError:
            location = storage.location
        try:
            f = storage.open(str(path.relative_to(location)))
            f.name = path.name
            yield f
        except (OSError, ValueError):
            logger.exception("File not found: %s", path)
//  [python]
def safe_text(raw_text: str) -> jinja2.Markup:
    """
    Sanitise text (escape any HTML tags), and then linkify any bare URLs.

    Args
        raw_text: Unsafe text which might include HTML markup.

    Returns:
        A Markup object ready to safely use in a Jinja template.
    """
    return jinja2.Markup(
        bleach.linkify(bleach.clean(raw_text, tags=[], attributes={}, strip=False))
    )
//  [python]
def safe_text(raw_text: str) -> jinja2.Markup:
    """
    Process text: treat it as HTML but escape any tags (ie. just escape the
    HTML) then linkify it.
    """
    return jinja2.Markup(
        bleach.linkify(bleach.clean(raw_text, tags=[], attributes={}, strip=False))
    )
//  [python]
def _check_put_dtypes(self, vals, indices=None):
  """Validate and convert `vals` to a list of `Tensor`s.

  The `vals` argument can be a Tensor, a list or tuple of tensors, or a
  dictionary with tensor values.

  If `vals` is a list, then the appropriate indices associated with the
  values must be provided.

  If it is a dictionary, the staging area must have been constructed with a
  `names` attribute and the dictionary keys must match the staging area names.
  `indices` will be inferred from the dictionary keys.
  If the staging area was constructed with a `names` attribute, `vals` must
  be a dictionary.

  Checks that the dtype and shape of each value matches that
  of the staging area.

  Args:
    vals: A tensor, a list or tuple of tensors, or a dictionary.

  Returns:
    A (tensors, indices) tuple where `tensors` is a list of `Tensor` objects
    and `indices` is a list of indices associated with the tensors.

  Raises:
    ValueError: If `vals` or `indices` is invalid.
  """
  if isinstance(vals, dict):
    if not self._names:
      raise ValueError(
          "Staging areas must have names to enqueue a dictionary")
    if not set(vals.keys()).issubset(self._names):
      raise ValueError("Keys in dictionary to put do not match names "
                       f"of staging area. Dictionary: {sorted(vals.keys())}"
                       f"Queue: {sorted(self._names)}")
    # The order of values in `self._names` indicates the order in which the
    # tensors in the dictionary `vals` must be listed.
    vals, indices, _ = zip(*[(vals[k], i, k)
                             for i, k in enumerate(self._names)
                             if k in vals])
  else:
    if self._names:
      raise ValueError("You must enqueue a dictionary in a staging area "
                       "with names")

    if indices is None:
      raise ValueError("Indices must be supplied when inserting a list "
                       "of tensors")

    if len(indices) != len(vals):
      raise ValueError(f"Number of indices {len(indices)} doesn't match "
                       f"number of values {len(vals)}")

    if not isinstance(vals, (list, tuple)):
      vals = [vals]
      indices = [0]

  # Sanity check number of values
  if not len(vals) <= len(self._dtypes):
    raise ValueError(f"Unexpected number of inputs {len(vals)} vs "
                     f"{len(self._dtypes)}")

  tensors = []

  for val, i in zip(vals, indices):
    dtype, shape = self._dtypes[i], self._shapes[i]
    # Check dtype
    if val.dtype != dtype:
      raise ValueError(f"Datatypes do not match. "
                       f"Received val.dtype {str(val.dtype)} and "
                       f"dtype {str(dtype)}")
    # Check shape
    val.get_shape().assert_is_compatible_with(shape)

    tensors.append(
        ops.convert_to_tensor(val, dtype=dtype, name="component_%d" % i))

  return tensors, indices
//  [python]
def _check_put_dtypes(self, vals, indices=None):
  """Validate and convert `vals` to a list of `Tensor`s.

  The `vals` argument can be a Tensor, a list or tuple of tensors, or a
  dictionary with tensor values.

  If `vals` is a list, then the appropriate indices associated with the
  values must be provided.

  If it is a dictionary, the staging area must have been constructed with a
  `names` attribute and the dictionary keys must match the staging area names.
  `indices` will be inferred from the dictionary keys.
  If the staging area was constructed with a `names` attribute, `vals` must
  be a dictionary.

  Checks that the dtype and shape of each value matches that
  of the staging area.

  Args:
    vals: A tensor, a list or tuple of tensors, or a dictionary.

  Returns:
    A (tensors, indices) tuple where `tensors` is a list of `Tensor` objects
    and `indices` is a list of indices associated with the tensors.

  Raises:
    ValueError: If `vals` or `indices` is invalid.
  """
  if isinstance(vals, dict):
    if not self._names:
      raise ValueError(
          "Staging areas must have names to enqueue a dictionary")
    if not set(vals.keys()).issubset(self._names):
      raise ValueError("Keys in dictionary to put do not match names "
                       f"of staging area. Dictionary: {sorted(vals.keys())}"
                       f"Queue: {sorted(self._names)}")
    # The order of values in `self._names` indicates the order in which the
    # tensors in the dictionary `vals` must be listed.
    vals, indices, _ = zip(*[(vals[k], i, k)
                             for i, k in enumerate(self._names)
                             if k in vals])
  else:
    if self._names:
      raise ValueError("You must enqueue a dictionary in a staging area "
                       "with names")

    if indices is None:
      raise ValueError("Indices must be supplied when inserting a list "
                       "of tensors")

    if len(indices) != len(vals):
      raise ValueError(f"Number of indices {len(indices)} doesn't match "
                       f"number of values {len(vals)}")

    if not isinstance(vals, (list, tuple)):
      vals = [vals]
      indices = [0]

  # Sanity check number of values
  if not len(vals) <= len(self._dtypes):
    raise ValueError(f"Unexpected number of inputs {len(vals)} vs"
                     f"{len(self._dtypes)}")

  tensors = []

  for val, i in zip(vals, indices):
    dtype, shape = self._dtypes[i], self._shapes[i]
    # Check dtype
    if val.dtype != dtype:
      raise ValueError(f"Datatypes do not match. "
                       f"Received val.dtype {str(val.dtype)} and "
                       f"dtype {str(dtype)}")
    # Check shape
    val.get_shape().assert_is_compatible_with(shape)

    tensors.append(
        ops.convert_to_tensor(val, dtype=dtype, name="component_%d" % i))

  return tensors, indices
//  [python]
def __init__(self, app, conf):
    self.app = app
    self.memcache_servers = conf.get('memcache_servers')
    serialization_format = conf.get('memcache_serialization_support')

    if not self.memcache_servers or serialization_format is None:
        path = os.path.join(conf.get('swift_dir', '/etc/swift'),
                            'memcache.conf')
        memcache_conf = ConfigParser()
        if memcache_conf.read(path):
            if not self.memcache_servers:
                try:
                    self.memcache_servers = \
                        memcache_conf.get('memcache', 'memcache_servers')
                except (NoSectionError, NoOptionError):
                    pass
            if serialization_format is None:
                try:
                    serialization_format = \
                        memcache_conf.get('memcache',
                                          'memcache_serialization_support')
                except (NoSectionError, NoOptionError):
                    pass

    if not self.memcache_servers:
        self.memcache_servers = '127.0.0.1:11211'
    if serialization_format is None:
        serialization_format = 2

    self.memcache = MemcacheRing(
        [s.strip() for s in self.memcache_servers.split(',') if s.strip()],
        allow_pickle=(serialization_format == 0),
        allow_unpickle=(serialization_format <= 1))
//  [python]
def __init__(self, app, conf):
    self.app = app
    self.memcache_servers = conf.get('memcache_servers')
    if not self.memcache_servers:
        path = os.path.join(conf.get('swift_dir', '/etc/swift'),
                            'memcache.conf')
        memcache_conf = ConfigParser()
        if memcache_conf.read(path):
            try:
                self.memcache_servers = \
                    memcache_conf.get('memcache', 'memcache_servers')
            except (NoSectionError, NoOptionError):
                pass
    if not self.memcache_servers:
        self.memcache_servers = '127.0.0.1:11211'
    self.memcache = MemcacheRing(
        [s.strip() for s in self.memcache_servers.split(',') if s.strip()])
//  [python]
def test_underscore_traversal(self):
    # Prevent traversal to names starting with an underscore (_)
    ec = self._makeContext()

    with self.assertRaises(NotFound):
        ec.evaluate("context/__class__")

    with self.assertRaises((NotFound, LocationError)):
        ec.evaluate("nocall: random/_itertools/repeat")

    with self.assertRaises((NotFound, LocationError)):
        ec.evaluate("random/_itertools/repeat/foobar")
//  [python]
def test_underscore_traversal(self):
    # Prevent traversal to names starting with an underscore (_)
    ec = self._makeContext()

    with self.assertRaises(NotFound):
        ec.evaluate("context/__class__")

    with self.assertRaises(NotFound):
        ec.evaluate("nocall: random/_itertools/repeat")

    with self.assertRaises(NotFound):
        ec.evaluate("random/_itertools/repeat/foobar")
//  [python]
def whitelist(allow_guest=False, xss_safe=False, methods=None):
	"""
	Decorator for whitelisting a function and making it accessible via HTTP.
	Standard request will be `/api/method/[path.to.method]`

	:param allow_guest: Allow non logged-in user to access this method.
	:param methods: Allowed http method to access the method.

	Use as:

		@frappe.whitelist()
		def myfunc(param1, param2):
			pass
	"""

	if not methods:
		methods = ['GET', 'POST', 'PUT', 'DELETE']

	def innerfn(fn):
		global whitelisted, guest_methods, xss_safe_methods, allowed_http_methods_for_whitelisted_func

		# get function from the unbound / bound method
		# this is needed because functions can be compared, but not methods
		if hasattr(fn, '__func__'):
			fn = fn.__func__

		whitelisted.append(fn)
		allowed_http_methods_for_whitelisted_func[fn] = methods

		if allow_guest:
			guest_methods.append(fn)

			if xss_safe:
				xss_safe_methods.append(fn)

		return fn

	return innerfn
//  [python]
def whitelist(allow_guest=False, xss_safe=False, methods=None):
	"""
	Decorator for whitelisting a function and making it accessible via HTTP.
	Standard request will be `/api/method/[path.to.method]`

	:param allow_guest: Allow non logged-in user to access this method.
	:param methods: Allowed http method to access the method.

	Use as:

		@frappe.whitelist()
		def myfunc(param1, param2):
			pass
	"""

	if not methods:
		methods = ['GET', 'POST', 'PUT', 'DELETE']

	def innerfn(fn):
		global whitelisted, guest_methods, xss_safe_methods, allowed_http_methods_for_whitelisted_func
		whitelisted.append(fn)

		allowed_http_methods_for_whitelisted_func[fn] = methods

		if allow_guest:
			guest_methods.append(fn)

			if xss_safe:
				xss_safe_methods.append(fn)

		return fn

	return innerfn
//  [python]
def default_dir():
    """ Return a default location to store compiled files and catalogs.

        XX is the Python version number in all paths listed below
        On windows, the default location is the temporary directory
        returned by gettempdir()/pythonXX.

        On Unix, ~/.pythonXX_compiled is the default location.  If it doesn't
        exist, it is created.  The directory is marked rwx------.

        If for some reason it isn't possible to build a default directory
        in the user's home, /tmp/<uid>_pythonXX_compiled is used.  If it
        doesn't exist, it is created.  The directory is marked rwx------
        to try and keep people from being able to sneak a bad module
        in on you. If the directory already exists in /tmp/ and is not
        secure, new one is created.
    """
    # Use a cached value for fast return if possible
    if hasattr(default_dir, "cached_path") and \
       check_dir(default_dir.cached_path):
        return default_dir.cached_path

    if sys.platform == 'win32':
        path = default_dir_win()
    else:
        path = default_dir_posix()

    # Cache the default dir path so that this function returns quickly after
    # being called once (nothing in it should change after the first call)
    default_dir.cached_path = path

    return path
//  [python]
def default_dir():
    """ Return a default location to store compiled files and catalogs.

        XX is the Python version number in all paths listed below
        On windows, the default location is the temporary directory
        returned by gettempdir()/pythonXX.

        On Unix, ~/.pythonXX_compiled is the default location.  If it doesn't
        exist, it is created.  The directory is marked rwx------.

        If for some reason it isn't possible to build a default directory
        in the user's home, /tmp/<uid>_pythonXX_compiled is used.  If it
        doesn't exist, it is created.  The directory is marked rwx------
        to try and keep people from being able to sneak a bad module
        in on you.

    """

    # Use a cached value for fast return if possible
    if hasattr(default_dir,"cached_path") and \
       os.path.exists(default_dir.cached_path) and \
       os.access(default_dir.cached_path, os.W_OK):
        return default_dir.cached_path

    python_name = "python%d%d_compiled" % tuple(sys.version_info[:2])
    path_candidates = []
    if sys.platform != 'win32':
        try:
            path_candidates.append(os.path.join(os.environ['HOME'],
                                                '.' + python_name))
        except KeyError:
            pass

        temp_dir = repr(os.getuid()) + '_' + python_name
        path_candidates.append(os.path.join(tempfile.gettempdir(), temp_dir))
    else:
        path_candidates.append(os.path.join(tempfile.gettempdir(),
                                           "%s" % whoami(), python_name))

    writable = False
    for path in path_candidates:
        if not os.path.exists(path):
            try:
                os.makedirs(path, mode=0o700)
            except OSError:
                continue
        if is_writable(path):
            writable = True
            break

    if not writable:
        print('warning: default directory is not write accessible.')
        print('default:', path)

    # Cache the default dir path so that this function returns quickly after
    # being called once (nothing in it should change after the first call)
    default_dir.cached_path = path

    return path
//  [python]
def intermediate_dir(tmp_dir=None):
    """
    Temporary directory for storing .cpp and .o files during builds.

    First, try to find the dir and if it exists, verify it is safe.
    Otherwise, create it.
    """
    im_dir = find_valid_temp_dir(intermediate_dir_prefix(), tmp_dir)
    py_im_dir = py_intermediate_dir()
    if im_dir is None:
        py_im_dir = py_intermediate_dir()
        im_dir = create_intermediate_dir(tmp_dir)
    else:
        im_dir = os.path.join(im_dir, py_im_dir)
        if not os.path.isdir(im_dir):
            os.mkdir(im_dir, 0o700)
    return im_dir
//  [python]
def intermediate_dir():
    """ Location in temp dir for storing .cpp and .o  files during
        builds.
    """
    python_name = "python%d%d_intermediate" % tuple(sys.version_info[:2])
    path = os.path.join(tempfile.gettempdir(),"%s"%whoami(),python_name)
    if not os.path.exists(path):
        os.makedirs(path, mode=0o700)
    return path
//  [python]
def _notify(self, assoc, attempt):
    """
    Sends data about a new association (and, if necessary, the associated invites)
    to the associated MXID's homeserver.

    :param assoc: The association to send down to the homeserver.
    :type assoc: dict[str, any]
    :param attempt: The number of previous attempts to send this association.
    :type attempt: int
    """
    mxid = assoc["mxid"]
    mxid_parts = mxid.split(":", 1)
    if len(mxid_parts) != 2:
        logger.error(
            "Can't notify on bind for unparseable mxid %s. Not retrying.",
            assoc["mxid"],
        )
        return

    post_url = "matrix://%s/_matrix/federation/v1/3pid/onbind" % (
        mxid_parts[1],
    )

    logger.info("Making bind callback to: %s", post_url)

    # Make a POST to the chosen Synapse server
    http_client = FederationHttpClient(self.sydent)
    try:
        response = yield http_client.post_json_get_nothing(post_url, assoc, {})
    except Exception as e:
        self._notifyErrback(assoc, attempt, e)
        return

    # If the request failed, try again with exponential backoff
    if response.code != 200:
        self._notifyErrback(
            assoc, attempt, "Non-OK error code received (%d)" % response.code
        )
    else:
        logger.info("Successfully notified on bind for %s" % (mxid,))

        # Skip the deletion step if instructed so by the config.
        if not self.sydent.delete_tokens_on_bind:
            return

        # Only remove sent tokens when they've been successfully sent.
        try:
            joinTokenStore = JoinTokenStore(self.sydent)
            joinTokenStore.deleteTokens(assoc["medium"], assoc["address"])
            logger.info(
                "Successfully deleted invite for %s from the store",
                assoc["address"],
            )
        except Exception as e:
            logger.exception(
                "Couldn't remove invite for %s from the store",
                assoc["address"],
            )
//  [python]
def test_post_broken_body():
    response = client.post(
        "/items/",
        headers={"content-type": "application/json"},
        data="{some broken json}",
    )
    assert response.status_code == 422, response.text
    assert response.json() == {
        "detail": [
            {
                "loc": ["body", 1],
                "msg": "Expecting property name enclosed in double quotes: line 1 column 2 (char 1)",
                "type": "value_error.jsondecode",
                "ctx": {
                    "msg": "Expecting property name enclosed in double quotes",
                    "doc": "{some broken json}",
                    "pos": 1,
                    "lineno": 1,
                    "colno": 2,
                },
            }
        ]
    }
//  [python]
def test_post_broken_body():
    response = client.post("/items/", data={"name": "Foo", "price": 50.5})
    assert response.status_code == 422, response.text
    assert response.json() == {
        "detail": [
            {
                "ctx": {
                    "colno": 1,
                    "doc": "name=Foo&price=50.5",
                    "lineno": 1,
                    "msg": "Expecting value",
                    "pos": 0,
                },
                "loc": ["body", 0],
                "msg": "Expecting value: line 1 column 1 (char 0)",
                "type": "value_error.jsondecode",
            }
        ]
    }
    with patch("json.loads", side_effect=Exception):
        response = client.post("/items/", json={"test": "test2"})
        assert response.status_code == 400, response.text
    assert response.json() == {"detail": "There was an error parsing the body"}
//  [python]
def is_writable(dir):
    """Determine whether a given directory is writable in a portable manner.

    Parameters
    ----------
    dir : str
        A string represeting a path to a directory on the filesystem.

    Returns
    -------
    res : bool
        True or False.
    """
    if not os.path.isdir(dir):
        return False

    # Do NOT use a hardcoded name here due to the danger from race conditions
    # on NFS when multiple processes are accessing the same base directory in
    # parallel.  We use both hostname and process id for the prefix in an
    # attempt to ensure that there can really be no name collisions (tempfile
    # appends 6 random chars to this prefix).
    prefix = 'dummy_%s_%s_' % (socket.gethostname(),os.getpid())
    try:
        tmp = tempfile.TemporaryFile(prefix=prefix,dir=dir)
    except OSError:
        return False
    # The underlying file is destroyed upon closing the file object (under
    # *nix, it was unlinked at creation time)
    tmp.close()
    return True
//  [python]
def is_writable(dir):
    """Determine whether a given directory is writable in a portable manner.

    Parameters
    ----------
    dir : str
        A string represeting a path to a directory on the filesystem.

    Returns
    -------
    res : bool
        True or False.
    """
    if not os.path.isdir(dir):
        return False

    # Do NOT use a hardcoded name here due to the danger from race conditions
    # on NFS when multiple processes are accessing the same base directory in
    # parallel.  We use both hostname and pocess id for the prefix in an
    # attempt to ensure that there can really be no name collisions (tempfile
    # appends 6 random chars to this prefix).
    prefix = 'dummy_%s_%s_' % (socket.gethostname(),os.getpid())
    try:
        tmp = tempfile.TemporaryFile(prefix=prefix,dir=dir)
    except OSError:
        return False
    # The underlying file is destroyed upon closing the file object (under
    # *nix, it was unlinked at creation time)
    tmp.close()
    return True
//  [python]
def innerfn(fn):
	global whitelisted, guest_methods, xss_safe_methods, allowed_http_methods_for_whitelisted_func

	# get function from the unbound / bound method
	# this is needed because functions can be compared, but not methods
	if hasattr(fn, '__func__'):
		fn = fn.__func__

	whitelisted.append(fn)
	allowed_http_methods_for_whitelisted_func[fn] = methods

	if allow_guest:
		guest_methods.append(fn)

		if xss_safe:
			xss_safe_methods.append(fn)

	return fn
//  [python]
def innerfn(fn):
	global whitelisted, guest_methods, xss_safe_methods, allowed_http_methods_for_whitelisted_func
	whitelisted.append(fn)

	allowed_http_methods_for_whitelisted_func[fn] = methods

	if allow_guest:
		guest_methods.append(fn)

		if xss_safe:
			xss_safe_methods.append(fn)

	return fn
//  [python]
def visit_Call(self, node):
    """ A couple function calls are supported: bson's ObjectId() and
    datetime().
    """
    if isinstance(node.func, ast.Name):
        if node.func.id == 'ObjectId':
            try:
                self.current_value = ObjectId(node.args[0].s)
            except:
                pass
        elif node.func.id == 'datetime':
            values = []
            for arg in node.args:
                values.append(arg.n)
            try:
                self.current_value = datetime(*values)
            except:
                pass
//  [python]
def visit_Call(self, node):
    """ A couple function calls are supported: bson's ObjectId() and
    datetime().
    """
    if isinstance(node.func, ast.Name):
        expr = None
        if node.func.id == 'ObjectId':
            expr = "('" + node.args[0].s + "')"
        elif node.func.id == 'datetime':
            values = []
            for arg in node.args:
                values.append(str(arg.n))
            expr = "(" + ", ".join(values) + ")"
        if expr:
            self.current_value = eval(node.func.id + expr)
//  [python]
def handler(request, operation, current_url):
    """Scheme handler for qute:// URLs.

    Args:
        request: QNetworkRequest to answer to.
        operation: The HTTP operation being done.
        current_url: The page we're on currently.

    Return:
        A QNetworkReply.
    """
    if operation != QNetworkAccessManager.GetOperation:
        return networkreply.ErrorNetworkReply(
            request, "Unsupported request type",
            QNetworkReply.ContentOperationNotPermittedError)

    url = request.url()

    if ((url.scheme(), url.host(), url.path()) ==
            ('qute', 'settings', '/set')):
        if current_url != QUrl('qute://settings/'):
            log.webview.warning("Blocking malicious request from {} to {}"
                                .format(current_url.toDisplayString(),
                                        url.toDisplayString()))
            return networkreply.ErrorNetworkReply(
                request, "Invalid qute://settings request",
                QNetworkReply.ContentAccessDenied)

    try:
        mimetype, data = qutescheme.data_for_url(url)
    except qutescheme.NoHandlerFound:
        errorstr = "No handler found for {}!".format(url.toDisplayString())
        return networkreply.ErrorNetworkReply(
            request, errorstr, QNetworkReply.ContentNotFoundError)
    except qutescheme.QuteSchemeOSError as e:
        return networkreply.ErrorNetworkReply(
            request, str(e), QNetworkReply.ContentNotFoundError)
    except qutescheme.QuteSchemeError as e:
        return networkreply.ErrorNetworkReply(request, e.errorstring, e.error)
    except qutescheme.Redirect as e:
        qtutils.ensure_valid(e.url)
        return networkreply.RedirectNetworkReply(e.url)

    return networkreply.FixedDataNetworkReply(request, data, mimetype)
//  [python]
def handler(request):
    """Scheme handler for qute:// URLs.

    Args:
        request: QNetworkRequest to answer to.

    Return:
        A QNetworkReply.
    """
    try:
        mimetype, data = qutescheme.data_for_url(request.url())
    except qutescheme.NoHandlerFound:
        errorstr = "No handler found for {}!".format(
            request.url().toDisplayString())
        return networkreply.ErrorNetworkReply(
            request, errorstr, QNetworkReply.ContentNotFoundError)
    except qutescheme.QuteSchemeOSError as e:
        return networkreply.ErrorNetworkReply(
            request, str(e), QNetworkReply.ContentNotFoundError)
    except qutescheme.QuteSchemeError as e:
        return networkreply.ErrorNetworkReply(request, e.errorstring, e.error)
    except qutescheme.Redirect as e:
        qtutils.ensure_valid(e.url)
        return networkreply.RedirectNetworkReply(e.url)

    return networkreply.FixedDataNetworkReply(request, data, mimetype)
//  [python]
def set_session_tracks(display_obj):
    """Save igv tracks as a session object. This way it's easy to verify that a user is requesting one of these files from remote_static view endpoint

    Args:
        display_obj(dict): A display object containing case name, list of genes, lucus and tracks
    """
    session_tracks = list(display_obj.get("reference_track", {}).values())
    for key, track_items in display_obj.items():
        if key not in ["tracks", "custom_tracks", "sample_tracks", "cloud_public_tracks"]:
            continue
        for track_item in track_items:
            session_tracks += list(track_item.values())

    session["igv_tracks"] = session_tracks